Skip to content

Commit 3cd5f73

Browse files
jf-techjf-tech
authored
ISSUE-144: allow schema object property name contains any character (#145)
Perviously we have a validation forcing all output object property name to be only alpha-numeric (plus underscore). Turns out we have use cases where the property name might contains special characters such as `'$'` or `'%'`, or `'.'`. Loosen up the restriction and dealing with all special character in property name (thus in FQDN) properly with escaping/unescaping.
1 parent 2c284f5 commit 3cd5f73

File tree

10 files changed

+18
-25
lines changed

10 files changed

+18
-25
lines changed

‎extensions/omniv21/samples/json/.snapshots/Test1_Single_Object

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"RawRecord": "{\"items\":[{\"item_price\":12.34,\"item_sku\":\"ab123\",\"number_purchased\":5},{\"item_price\":3.12,\"item_sku\":\"ck763-23\",\"number_purchased\":2}],\"order_id\":\"1234567\",\"tracking_number\":\"1z9999999999999999\"}",
44
"RawRecordHash": "6ef8f474-8b41-3366-9356-ef44a84c1439",
55
"TransformedRecord": {
6+
"$order.id": "1234567",
67
"items": [
78
{
89
"sku": "AB123",
@@ -13,7 +14,6 @@
1314
"total_price": 6.24
1415
}
1516
],
16-
"order_id": "1234567",
1717
"tracking_number": "1Z9999999999999999"
1818
}
1919
}

‎extensions/omniv21/samples/json/1_single_object.schema.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
},
66
"transform_declarations": {
77
"FINAL_OUTPUT": { "object": {
8-
"order_id": { "xpath": "order_id" },
8+
"$order.id": { "xpath": "order_id" },
99
"tracking_number": { "custom_func": {
1010
"name": "upper",
1111
"args": [ { "xpath": "tracking_number" } ]

‎extensions/omniv21/transform/.snapshots/TestValidateTransformDeclarations-success

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
{
22
"object": {
3+
"$field_13 with space. and other non-alphanumeric chars": {
4+
"custom_parse": "test_custom_parse",
5+
"fqdn": "FINAL_OUTPUT.$field_13 with space%. and other non-alphanumeric chars",
6+
"kind": "custom_parse",
7+
"parent": "FINAL_OUTPUT"
8+
},
39
"field1": {
410
"const": "value1",
511
"fqdn": "FINAL_OUTPUT.field1",
@@ -200,12 +206,6 @@
200206
],
201207
"parent": "FINAL_OUTPUT"
202208
},
203-
"field_13": {
204-
"custom_parse": "test_custom_parse",
205-
"fqdn": "FINAL_OUTPUT.field_13",
206-
"kind": "custom_parse",
207-
"parent": "FINAL_OUTPUT"
208-
},
209209
"field_9": {
210210
"xpath": "1/2/3",
211211
"object": {
@@ -227,14 +227,14 @@
227227
"fqdn": "FINAL_OUTPUT",
228228
"kind": "object",
229229
"children": [
230+
"FINAL_OUTPUT.$field_13 with space%. and other non-alphanumeric chars",
230231
"FINAL_OUTPUT.field1",
231232
"FINAL_OUTPUT.field2",
232233
"FINAL_OUTPUT.field3",
233234
"FINAL_OUTPUT.field6",
234235
"FINAL_OUTPUT.field_10",
235236
"FINAL_OUTPUT.field_11",
236237
"FINAL_OUTPUT.field_12",
237-
"FINAL_OUTPUT.field_13",
238238
"FINAL_OUTPUT.field_9"
239239
],
240240
"parent": "(nil)"

‎extensions/omniv21/transform/parse.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ func (p *parseCtx) parseObject(n *idr.Node, decl *Decl) (interface{}, error) {
211211
// value returned by p.ParseNode is already normalized, thus this
212212
// normalizeAndSaveValue won't fail.
213213
_ = normalizeAndSaveValue(childDecl, childValue, func(normalizedValue interface{}) {
214-
obj[strs.LastNameletOfFQDN(childDecl.fqdn)] = normalizedValue
214+
obj[strs.LastNameletOfFQDNWithEsc(childDecl.fqdn)] = normalizedValue
215215
})
216216
}
217217
return normalizeAndReturnValue(decl, obj)

‎extensions/omniv21/transform/validate.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ func (ctx *validateCtx) validateXPath(fqdn string, decl *Decl, templateRefStack
104104
func (ctx *validateCtx) validateObject(fqdn string, decl *Decl, templateRefStack []string) error {
105105
for childName, childDecl := range decl.Object {
106106
childDecl, err := ctx.validateDecl(
107-
strs.BuildFQDN(fqdn, childName), childDecl, templateRefStack)
107+
// childName can contain '.' or '%', it needs to be escaped.
108+
strs.BuildFQDN(fqdn, strs.BuildFQDNWithEsc(childName)), childDecl, templateRefStack)
108109
if err != nil {
109110
return err
110111
}

‎extensions/omniv21/transform/validate_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func TestValidateTransformDeclarations(t *testing.T) {
4949
"field_10": { "xpath_dynamic": { "const": "X/Y/Z" }, "template": "template10" },
5050
"field_11": { "template": "template11" },
5151
"field_12": { "template": "template12" },
52-
"field_13": { "custom_parse": "test_custom_parse" }
52+
"$field_13 with space. and other non-alphanumeric chars": { "custom_parse": "test_custom_parse" }
5353
}},
5454
"template9": { "xpath": "1/2/3", "object": {
5555
"field9": { "xpath": "4/5/6" }

‎extensions/omniv21/validation/transformDeclarations.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎extensions/omniv21/validation/transformDeclarations.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@
8484
"value_object": {
8585
"type": "object",
8686
"patternProperties": {
87-
"^[_a-zA-Z0-9]+$": {
87+
"^.+$": {
8888
"oneOf": [
8989
{ "$ref": "#/definitions/const" },
9090
{ "$ref": "#/definitions/external" },

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ require (
1111
github.com/go-chi/chi v4.1.2+incompatible
1212
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
1313
github.com/google/uuid v1.1.2
14-
github.com/jf-tech/go-corelib v0.0.13
14+
github.com/jf-tech/go-corelib v0.0.14
1515
github.com/spf13/cobra v1.0.0
1616
github.com/spf13/pflag v1.0.5 // indirect
1717
github.com/stretchr/testify v1.6.1

‎go.sum

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
55
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
66
github.com/antchfx/xmlquery v1.3.1 h1:nIKWdtnhrXtj0/IRUAAw2I7TfpHUa3zMnHvNmPXFg+w=
77
github.com/antchfx/xmlquery v1.3.1/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
8-
github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg=
98
github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
109
github.com/antchfx/xpath v1.1.11 h1:WOFtK8TVAjLm3lbgqeP0arlHpvCEeTANeWZ/csPpJkQ=
1110
github.com/antchfx/xpath v1.1.11/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
@@ -22,7 +21,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
2221
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
2322
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
2423
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
25-
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
2624
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2725
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2826
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -64,8 +62,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
6462
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
6563
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
6664
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
67-
github.com/jf-tech/go-corelib v0.0.13 h1:4aDwS09bdfRb/loU6Va+kvqwNJq5irR5WjKVpzJxdag=
68-
github.com/jf-tech/go-corelib v0.0.13/go.mod h1:0+Fejzd53JtexKE5VI8I06WiBNATLIURRJgPrv4Yysg=
65+
github.com/jf-tech/go-corelib v0.0.14 h1:PXS6ApXGhZk+9TTxVFiQe9YYJV5liAzjmvotY7l4dCA=
66+
github.com/jf-tech/go-corelib v0.0.14/go.mod h1:0+Fejzd53JtexKE5VI8I06WiBNATLIURRJgPrv4Yysg=
6967
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
7068
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
7169
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -107,7 +105,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
107105
github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
108106
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
109107
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
110-
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
111108
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
112109
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
113110
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -145,7 +142,6 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
145142
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
146143
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
147144
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
148-
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
149145
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
150146
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
151147
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -159,9 +155,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
159155
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
160156
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
161157
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
162-
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
163158
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
164-
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
165159
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
166160
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
167161
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -175,14 +169,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
175169
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
176170
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
177171
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
178-
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
179172
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
180173
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
181174
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
182175
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
183176
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
184177
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
185-
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
186178
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
187179
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
188180
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

0 commit comments

Comments
 (0)