Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit d7f9681

Browse files
authored
Introduce json schema validation (#25)
Given the complexity of omniparser schemas (all of which are written in json), we'll use json schema to validate these schemas :) Created `schemavalidate` package for generating each schema section json schema and helper for doing json validation. Added `parser_settings` validation to parser.go Added `transform_declarations` validation to omniv2/plugin.go
1 parent 043e2f8 commit d7f9681

File tree

13 files changed

+684
-7
lines changed

13 files changed

+684
-7
lines changed

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ require (
1010
github.com/hashicorp/golang-lru v0.5.4
1111
github.com/jf-tech/iohelper v1.0.3
1212
github.com/stretchr/testify v1.6.1
13+
github.com/xeipuuv/gojsonschema v1.2.0
1314
golang.org/x/text v0.3.0
1415
)

‎go.sum‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,16 @@ github.com/jf-tech/iohelper v1.0.3/go.mod h1:X28R+KF0lnKEhZ8Q0iBzLI9FKHJy/jXZ+ax
1717
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
1818
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1919
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
20+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
2021
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
2122
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
2223
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
24+
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
25+
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
26+
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
27+
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
28+
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
29+
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
2330
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
2431
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
2532
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

‎gogen.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
function green_printf () {
4+
printf "\e[32m$@\e[m\n"
5+
}
6+
7+
function red_printf () {
8+
printf "\e[31m$@\e[m\n"
9+
}
10+
11+
function panic () {
12+
echo
13+
red_printf "$@"
14+
echo
15+
exit 1
16+
}
17+
18+
function panic_fail_op () {
19+
panic "Operation failed! Exit."
20+
}
21+
22+
SCRIPT_DIR=$(pwd `dirname "0ドル"`)
23+
24+
cd $SCRIPT_DIR/omniparser/schemavalidate || panic_fail_op
25+
green_printf "go:generate in 'omniparser/schemavalidate'..."
26+
go generate || panic_fail_op
27+
28+
cd $SCRIPT_DIR
29+
30+
echo
31+
green_printf "go generate completed!\n"

‎omniparser/parser.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/jf-tech/omniparser/omniparser/errs"
1313
"github.com/jf-tech/omniparser/omniparser/schemaplugin"
1414
omniv2 "github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2"
15+
"github.com/jf-tech/omniparser/omniparser/schemavalidate"
1516
"github.com/jf-tech/omniparser/omniparser/transformctx"
1617
)
1718

@@ -53,12 +54,16 @@ func NewParser(schemaName string, schemaReader io.Reader, pluginConfigs ...Schem
5354
if err != nil {
5455
return nil, fmt.Errorf("unable to read schema '%s': %s", schemaName, err.Error())
5556
}
56-
varschemaHeader schemaplugin.Header
57-
err = json.Unmarshal(schemaContent, &schemaHeader)
57+
// validate the universal parser_settings header schema.
58+
err = schemavalidate.SchemaValidate(schemaName, schemaContent, schemavalidate.JSONSchemaParserSettings)
5859
if err != nil {
59-
returnnil, fmt.Errorf(
60-
"unable to read schema '%s': corrupted header `parser_settings`: %s", schemaName, err)
60+
// The err from schemavalidate.SchemaValidate is already context formatted.
61+
returnnil, err
6162
}
63+
var schemaHeader schemaplugin.Header
64+
// parser_settings has just been json schema validated. so unmarshaling will not go wrong.
65+
_ = json.Unmarshal(schemaContent, &schemaHeader)
66+
6267
var allPluginConfigs []SchemaPluginConfig
6368
allPluginConfigs = append(allPluginConfigs, pluginConfigs...)
6469
allPluginConfigs = append(allPluginConfigs, SchemaPluginConfig{

‎omniparser/parser_test.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func TestNewParser(t *testing.T) {
3232
name: "fail to unmarshal schema header",
3333
schema: "[invalid",
3434
pluginCfgs: nil,
35-
expectedErr: "unable to read schema 'test-schema': corrupted header `parser_settings`:.*",
35+
expectedErr: "unable to perform schema validation: invalid character 'i' looking for beginning of value",
3636
},
3737
{
3838
name: "no supported schema plugin",
@@ -48,7 +48,13 @@ func TestNewParser(t *testing.T) {
4848
expectedErr: errs.ErrSchemaNotSupported.Error(),
4949
},
5050
{
51-
name: "supported schema plugin found, but schema validation fails",
51+
name: "supported schema plugin found, but json schema validation for parser_settings failed",
52+
schema: `{"parser_settings": {"versionx": "9999", "file_format_type": "exe" }}`,
53+
pluginCfgs: nil,
54+
expectedErr: "schema 'test-schema' validation failed:\nparser_settings: version is required\nparser_settings: Additional property versionx is not allowed",
55+
},
56+
{
57+
name: "supported schema plugin found, but schema validation failed",
5258
schema: `{"parser_settings": {"version": "9999", "file_format_type": "exe" }}`,
5359
pluginCfgs: []SchemaPluginConfig{
5460
{
@@ -96,7 +102,7 @@ func TestNewParser(t *testing.T) {
96102
plugin, err := NewParser("test-schema", schemaReader, test.pluginCfgs...)
97103
if test.expectedErr != "" {
98104
assert.Error(t, err)
99-
assert.Regexp(t, test.expectedErr, err.Error())
105+
assert.Equal(t, test.expectedErr, err.Error())
100106
assert.Nil(t, plugin)
101107
} else {
102108
assert.NoError(t, err)

‎omniparser/schemaplugin/omni/v2/plugin.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/jf-tech/omniparser/omniparser/schemaplugin"
88
omniv2fileformat "github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2/fileformat"
99
omniv2xml "github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2/fileformat/xml"
10+
"github.com/jf-tech/omniparser/omniparser/schemavalidate"
1011
"github.com/jf-tech/omniparser/omniparser/transformctx"
1112
)
1213

@@ -23,6 +24,12 @@ func ParseSchema(ctx *schemaplugin.ParseSchemaCtx) (schemaplugin.Plugin, error)
2324
if ctx.Header.ParserSettings.Version != PluginVersion {
2425
return nil, errs.ErrSchemaNotSupported
2526
}
27+
// Now do transform_declarations json schema validation
28+
err := schemavalidate.SchemaValidate(ctx.Name, ctx.Content, schemavalidate.JSONSchemaTransformDeclarations)
29+
if err != nil {
30+
// err is always context formatted.
31+
return nil, err
32+
}
2633
// If caller specifies a custom FileFormat, we'll use it (and it only);
2734
// otherwise we'll use the builtin ones.
2835
fileFormats := []omniv2fileformat.FileFormat{

‎omniparser/schemaplugin/omni/v2/plugin_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,32 @@ func TestParseSchema_FormatNotSupported(t *testing.T) {
7676
FileFormatType: "unknown",
7777
},
7878
},
79+
Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
7980
})
8081
assert.Error(t, err)
8182
assert.Equal(t, errs.ErrSchemaNotSupported, err)
8283
assert.Nil(t, p)
8384
}
8485

86+
func TestParseSchema_TransformDeclarationsValidationFailed(t *testing.T) {
87+
p, err := ParseSchema(
88+
&schemaplugin.ParseSchemaCtx{
89+
Name: "test-schema",
90+
Header: schemaplugin.Header{
91+
ParserSettings: schemaplugin.ParserSettings{
92+
Version: PluginVersion,
93+
FileFormatType: "xml",
94+
},
95+
},
96+
Content: []byte(`{"transform_declarations": {}}`),
97+
})
98+
assert.Error(t, err)
99+
assert.Equal(t,
100+
`schema 'test-schema' validation failed: transform_declarations: FINAL_OUTPUT is required`,
101+
err.Error())
102+
assert.Nil(t, p)
103+
}
104+
85105
func TestParseSchema_CustomFileFormat_FormatNotSupported(t *testing.T) {
86106
p, err := ParseSchema(
87107
&schemaplugin.ParseSchemaCtx{
@@ -90,6 +110,7 @@ func TestParseSchema_CustomFileFormat_FormatNotSupported(t *testing.T) {
90110
Version: PluginVersion,
91111
},
92112
},
113+
Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
93114
PluginParams: &PluginParams{
94115
CustomFileFormat: testFileFormat{
95116
validateSchemaErr: errs.ErrSchemaNotSupported,
@@ -109,6 +130,7 @@ func TestParseSchema_CustomFileFormat_ValidationFailure(t *testing.T) {
109130
Version: PluginVersion,
110131
},
111132
},
133+
Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
112134
PluginParams: &PluginParams{
113135
CustomFileFormat: testFileFormat{
114136
validateSchemaErr: errors.New("validation failure"),
@@ -128,6 +150,7 @@ func TestParseSchema_CustomFileFormat_Success(t *testing.T) {
128150
Version: PluginVersion,
129151
},
130152
},
153+
Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
131154
PluginParams: &PluginParams{
132155
CustomFileFormat: testFileFormat{
133156
validateSchemaRuntime: "runtime data",
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"io/ioutil"
6+
"os"
7+
"text/template"
8+
)
9+
10+
const schemaTemplate = `// Code generated - DO NOT EDIT.
11+
12+
package schemavalidate
13+
14+
const (
15+
{{.SchemaVarName}} =
16+
` + "\x60" /*\x60 is this char '`' :) */ + `
17+
{{.Schema}}
18+
` + "\x60" + `
19+
)
20+
`
21+
22+
type templateVars struct {
23+
SchemaVarName string
24+
Schema string
25+
}
26+
27+
func main() {
28+
var jsonFileName string
29+
flag.StringVar(&jsonFileName, "json", "", "The name of json schema file.")
30+
var tv templateVars
31+
flag.StringVar(&tv.SchemaVarName, "varname", "", "The variable name of json schema string.")
32+
flag.Parse()
33+
34+
jsonFileContent, err := ioutil.ReadFile("./" + jsonFileName)
35+
if err != nil {
36+
os.Exit(1)
37+
}
38+
tv.Schema = string(jsonFileContent)
39+
40+
err = template.Must(template.New("genjsonschema").Parse(schemaTemplate)).Execute(os.Stdout, tv)
41+
if err != nil {
42+
os.Exit(1)
43+
}
44+
}
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"$id": "github.com/jf-tech/omniparser:parser_settings",
4+
"title": "omniparser schema: parser_settings",
5+
"type": "object",
6+
"properties": {
7+
"parser_settings": {
8+
"type": "object",
9+
"properties": {
10+
"version": { "type": "string" },
11+
"file_format_type": { "type": "string" },
12+
"encoding": {
13+
"type": "string",
14+
"enum": [ "utf-8", "iso-8859-1", "windows-1252" ]
15+
}
16+
},
17+
"required": [ "version", "file_format_type" ],
18+
"additionalProperties": false
19+
}
20+
},
21+
"required": [ "parser_settings" ]
22+
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /