fix: local seed consistency between fields (#235)

* fix: local seed consistency between fields * test: local seed consistency between fields
CGI-FR · Jun 14, 2023 · 5fb95bf · 5fb95bf
1 parent 85232a3
commit 5fb95bf
Show file tree

Hide file tree

Showing 23 changed files with 91 additions and 22 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,10 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [1.18.1]
+
+- `Fixed` local seed parameter should be consistent between different jsonpath
+
 ## [1.18.0]
 
 - `Added` `randomChoiceInCSV` and `hashInCSV` mask to get values from CSV files

diff --git a/pkg/hashcsv/hashcsv.go b/pkg/hashcsv/hashcsv.go
@@ -131,13 +131,15 @@ func (mrl MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
+	seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 	// set differents seeds for differents jsonpath
 	h := fnv.New64a()
 	h.Write([]byte(conf.Masking.Selector.Jsonpath))
 	conf.Seed += int64(h.Sum64())
 
 	if len(conf.Masking.Mask.HashInCSV.URI) != 0 {
-		mask, err := NewMask(conf.Masking.Mask.HashInCSV, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed))
+		mask, err := NewMask(conf.Masking.Mask.HashInCSV, conf.Seed, seeder)
 		return mask, true, err
 	}
 	return nil, false, nil

diff --git a/pkg/randdate/randdate.go b/pkg/randdate/randdate.go
@@ -62,11 +62,13 @@ func (dateRange MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (mo
 // Create a mask from a configuration
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
 	if conf.Masking.Mask.RandDate.DateMin != conf.Masking.Mask.RandDate.DateMax {
+		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 		// set differents seeds for differents jsonpath
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
-		return NewMask(conf.Masking.Mask.RandDate.DateMin, conf.Masking.Mask.RandDate.DateMax, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)), true, nil
+		return NewMask(conf.Masking.Mask.RandDate.DateMin, conf.Masking.Mask.RandDate.DateMax, conf.Seed, seeder), true, nil
 	}
 	return nil, false, nil
 }

diff --git a/pkg/randdura/randdura.go b/pkg/randdura/randdura.go
@@ -105,11 +105,14 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 
 // Create a mask from a configuration
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
-	if len(conf.Masking.Mask.RandomDuration.Min) != 0 || len(conf.Masking.Mask.RandomDuration.Max) != 0 { // set differents seeds for differents jsonpath
+	if len(conf.Masking.Mask.RandomDuration.Min) != 0 || len(conf.Masking.Mask.RandomDuration.Max) != 0 {
+		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
+		// set differents seeds for differents jsonpath
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
-		mask, err := NewMask(conf.Masking.Mask.RandomDuration.Min, conf.Masking.Mask.RandomDuration.Max, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed))
+		mask, err := NewMask(conf.Masking.Mask.RandomDuration.Min, conf.Masking.Mask.RandomDuration.Max, conf.Seed, seeder)
 		if err != nil {
 			return nil, false, err
 		}

diff --git a/pkg/randomcsv/randomcsv.go b/pkg/randomcsv/randomcsv.go
@@ -122,13 +122,15 @@ func (mrl MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
+	seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 	// set differents seeds for differents jsonpath
 	h := fnv.New64a()
 	h.Write([]byte(conf.Masking.Selector.Jsonpath))
 	conf.Seed += int64(h.Sum64())
 
 	if len(conf.Masking.Mask.RandomChoiceInCSV.URI) != 0 {
-		mask, err := NewMask(conf.Masking.Mask.RandomChoiceInCSV, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed))
+		mask, err := NewMask(conf.Masking.Mask.RandomChoiceInCSV, conf.Seed, seeder)
 		return mask, true, err
 	}
 	return nil, false, nil

diff --git a/pkg/randomdecimal/randomdecimal.go b/pkg/randomdecimal/randomdecimal.go
@@ -66,11 +66,13 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
 	if conf.Masking.Mask.RandomDecimal.Precision != 0 {
+		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 		// set differents seeds for differents jsonpath
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
-		return NewMask(conf.Masking.Mask.RandomDecimal.Min, conf.Masking.Mask.RandomDecimal.Max, conf.Masking.Mask.RandomDecimal.Precision, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)), true, nil
+		return NewMask(conf.Masking.Mask.RandomDecimal.Min, conf.Masking.Mask.RandomDecimal.Max, conf.Masking.Mask.RandomDecimal.Precision, conf.Seed, seeder), true, nil
 	}
 	return nil, false, nil
 }

diff --git a/pkg/randomint/randomint.go b/pkg/randomint/randomint.go
@@ -57,11 +57,12 @@ func (rim MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
 	if conf.Masking.Mask.RandomInt.Min != 0 || conf.Masking.Mask.RandomInt.Max != 0 {
+		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 		// set differents seeds for differents jsonpath
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
-		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
 		return NewMask(conf.Masking.Mask.RandomInt.Min, conf.Masking.Mask.RandomInt.Max, conf.Seed, seeder), true, nil
 	}
 	return nil, false, nil

diff --git a/pkg/randomlist/randomlist.go b/pkg/randomlist/randomlist.go
@@ -57,13 +57,15 @@ func (mrl MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
+	seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 	// set differents seeds for differents jsonpath
 	h := fnv.New64a()
 	h.Write([]byte(conf.Masking.Selector.Jsonpath))
 	conf.Seed += int64(h.Sum64())
 
 	if len(conf.Masking.Mask.RandomChoice) != 0 {
-		return NewMask(conf.Masking.Mask.RandomChoice, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)), true, nil
+		return NewMask(conf.Masking.Mask.RandomChoice, conf.Seed, seeder), true, nil
 	}
 
 	return nil, false, nil

diff --git a/pkg/randomuri/randomuri.go b/pkg/randomuri/randomuri.go
@@ -82,13 +82,15 @@ func (mrl MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
+	seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 	// set differents seeds for differents jsonpath
 	h := fnv.New64a()
 	h.Write([]byte(conf.Masking.Selector.Jsonpath))
 	conf.Seed += int64(h.Sum64())
 
 	if len(conf.Masking.Mask.RandomChoiceInURI) != 0 {
-		mask, err := NewMask(conf.Masking.Mask.RandomChoiceInURI, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed))
+		mask, err := NewMask(conf.Masking.Mask.RandomChoiceInURI, conf.Seed, seeder)
 		return mask, true, err
 	}
 	return nil, false, nil

diff --git a/pkg/regex/regex.go b/pkg/regex/regex.go
@@ -65,12 +65,14 @@ func (rm MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
 	if len(conf.Masking.Mask.Regex) != 0 {
+		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 		// set differents seeds for differents jsonpath
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
 
-		mask, err := NewMask(conf.Masking.Mask.Regex, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed))
+		mask, err := NewMask(conf.Masking.Mask.Regex, conf.Seed, seeder)
 		if err != nil {
 			return nil, true, err
 		}

diff --git a/pkg/transcode/transcode.go b/pkg/transcode/transcode.go
@@ -71,12 +71,13 @@ func (mrl MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 
 // Factory create a mask from a yaml config
 func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) {
+	seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 	// set differents seeds for differents jsonpath
 	if conf.Masking.Mask.Transcode != nil {
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
-		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
 		if classes := conf.Masking.Mask.Transcode.Classes; len(classes) > 0 {
 			return NewMask(classes, conf.Seed, seeder), true, nil
 		}

diff --git a/pkg/weightedchoice/weightedchoice.go b/pkg/weightedchoice/weightedchoice.go
@@ -105,11 +105,13 @@ func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error
 			maskWeight = append(maskWeight, model.WeightedChoiceType{Choice: v.Choice, Weight: v.Weight})
 		}
 
+		seeder := model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)
+
 		// set differents seeds for differents jsonpath
 		h := fnv.New64a()
 		h.Write([]byte(conf.Masking.Selector.Jsonpath))
 		conf.Seed += int64(h.Sum64())
-		return NewMask(maskWeight, conf.Seed, model.NewSeeder(conf.Masking.Seed.Field, conf.Seed)), true, nil
+		return NewMask(maskWeight, conf.Seed, seeder), true, nil
 	}
 	return nil, false, nil
 }

diff --git a/test/suites/masking_randdate.yml b/test/suites/masking_randdate.yml
@@ -97,5 +97,5 @@ testcases:
         echo -e '{"date":1}\n{"date":2}\n{"date":3}\n{"date":2}\n{"date":1}\n{"date":3}' | TZ="" pimo | head -c -1 | tr '\n' ' '
       assertions:
       - result.code ShouldEqual 0
-      - result.systemout ShouldEqual {"date":"1997-10-30T16:13:09Z"} {"date":"1997-09-15T01:31:05Z"} {"date":"1997-05-30T12:42:07Z"} {"date":"1997-09-15T01:31:05Z"} {"date":"1997-10-30T16:13:09Z"} {"date":"1997-05-30T12:42:07Z"}
+      - result.systemout ShouldEqual {"date":"1997-07-31T05:16:11Z"} {"date":"1997-02-07T18:24:46Z"} {"date":"1997-04-21T22:05:41Z"} {"date":"1997-02-07T18:24:46Z"} {"date":"1997-07-31T05:16:11Z"} {"date":"1997-04-21T22:05:41Z"}
       - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_random.yml b/test/suites/masking_random.yml
@@ -70,5 +70,5 @@ testcases:
       echo -e '{"name":1}\n{"name":2}\n{"name":3}\n{"name":2}\n{"name":1}\n{"name":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"name":"Marcel"} {"name":"Marcel"} {"name":"Mickael"} {"name":"Marcel"} {"name":"Marcel"} {"name":"Mickael"}
+    - result.systemout ShouldEqual {"name":"Mickael"} {"name":"Marcel"} {"name":"Mathieu"} {"name":"Marcel"} {"name":"Mickael"} {"name":"Mathieu"}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_random_duration.yml b/test/suites/masking_random_duration.yml
@@ -102,5 +102,5 @@ testcases:
       echo -e '{"index": 1, "date": "2020-01-01T00:00:00Z"}\n{"index": 2, "date": "2020-01-01T00:00:00Z"}\n{"index": 1, "date": "2020-01-01T00:00:00Z"}\n{"index": 2, "date": "2020-01-01T00:00:00Z"}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
       - result.code ShouldEqual 0
-      - result.systemout ShouldEqual {"date":"2019-12-29T22:49:04.208101611Z"} {"date":"2019-12-25T10:44:05.019037735Z"} {"date":"2019-12-29T22:49:04.208101611Z"} {"date":"2019-12-25T10:44:05.019037735Z"}
+      - result.systemout ShouldEqual {"date":"2019-12-28T07:57:54.421699429Z"} {"date":"2019-12-10T09:05:42.752967314Z"} {"date":"2019-12-28T07:57:54.421699429Z"} {"date":"2019-12-10T09:05:42.752967314Z"}
       - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_random_in_csv.yml b/test/suites/masking_random_in_csv.yml
@@ -94,7 +94,7 @@ testcases:
       echo -e '{"person":1}\n{"person":2}\n{"person":3}\n{"person":2}\n{"person":1}\n{"person":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"person":"Luce"} {"person":"Patricia"} {"person":"Luce"} {"person":"Patricia"} {"person":"Luce"} {"person":"Luce"}
+    - result.systemout ShouldEqual {"person":"Anissa"} {"person":"Patricia"} {"person":"Jean-Claude"} {"person":"Patricia"} {"person":"Anissa"} {"person":"Jean-Claude"}
     - result.systemerr ShouldBeEmpty
 
 - name: random csv with different separator

diff --git a/test/suites/masking_random_in_uri.yml b/test/suites/masking_random_in_uri.yml
@@ -99,5 +99,5 @@ testcases:
       echo -e '{"name":1}\n{"name":2}\n{"name":3}\n{"name":2}\n{"name":1}\n{"name":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"name":"Benjamin"} {"name":"Benjamin"} {"name":"Mickael"} {"name":"Benjamin"} {"name":"Benjamin"} {"name":"Mickael"}
+    - result.systemout ShouldEqual {"name":"Mickael"} {"name":"Benjamin"} {"name":"Marc"} {"name":"Benjamin"} {"name":"Mickael"} {"name":"Marc"}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_random_int.yml b/test/suites/masking_random_int.yml
@@ -96,5 +96,5 @@ testcases:
       echo -e '{"age":1}\n{"age":2}\n{"age":3}\n{"age":2}\n{"age":1}\n{"age":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"age":87} {"age":69} {"age":65} {"age":69} {"age":87} {"age":65}
+    - result.systemout ShouldEqual {"age":82} {"age":81} {"age":56} {"age":81} {"age":82} {"age":56}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_randomdecimal.yml b/test/suites/masking_randomdecimal.yml
@@ -103,5 +103,5 @@ testcases:
       echo -e '{"amount":1}\n{"amount":2}\n{"amount":3}\n{"amount":2}\n{"amount":1}\n{"amount":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"amount":6.926} {"amount":2.042} {"amount":0.512} {"amount":2.042} {"amount":6.926} {"amount":0.512}
+    - result.systemout ShouldEqual {"amount":6.367} {"amount":1.186} {"amount":8.9} {"amount":1.186} {"amount":6.367} {"amount":8.9}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_regex.yml b/test/suites/masking_regex.yml
@@ -87,5 +87,5 @@ testcases:
       echo -e '{"name":1}\n{"name":2}\n{"name":3}\n{"name":2}\n{"name":1}\n{"name":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"name":"rvktt"} {"name":"bjrxd"} {"name":"qlofi"} {"name":"bjrxd"} {"name":"rvktt"} {"name":"qlofi"}
+    - result.systemout ShouldEqual {"name":"bakvg"} {"name":"vvauw"} {"name":"cdjgf"} {"name":"vvauw"} {"name":"bakvg"} {"name":"cdjgf"}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_transcode.yml b/test/suites/masking_transcode.yml
@@ -66,5 +66,5 @@ testcases:
       echo -e '{"id":"1A"}\n{"id":"2B"}\n{"id":"3C"}\n{"id":"2B"}\n{"id":"1A"}\n{"id":"3C"}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"id":"0E"} {"id":"3S"} {"id":"3Q"} {"id":"3S"} {"id":"0E"} {"id":"3Q"}
+    - result.systemout ShouldEqual {"id":"3I"} {"id":"7C"} {"id":"3D"} {"id":"7C"} {"id":"3I"} {"id":"3D"}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/masking_weighted.yml b/test/suites/masking_weighted.yml
@@ -107,5 +107,5 @@ testcases:
       echo -e '{"field":1}\n{"field":2}\n{"field":3}\n{"field":2}\n{"field":1}\n{"field":3}' | pimo | head -c -1 | tr '\n' ' '
     assertions:
     - result.code ShouldEqual 0
-    - result.systemout ShouldEqual {"field":"three"} {"field":"un"} {"field":"un"} {"field":"un"} {"field":"three"} {"field":"un"}
+    - result.systemout ShouldEqual {"field":"un"} {"field":"three"} {"field":"two"} {"field":"three"} {"field":"un"} {"field":"two"}
     - result.systemerr ShouldBeEmpty
diff --git a/test/suites/set_seed.yml b/test/suites/set_seed.yml
@@ -83,3 +83,47 @@ testcases:
           - result.code ShouldEqual 0
           - result.systemoutjson.name ShouldEqual Rolande
           - result.systemerr ShouldBeEmpty
+
+  - name: test local seed consistency between fields
+    steps:
+      - script: rm -f masking.yml
+      - script: |-
+          cat > masking.yml <<EOF
+          version: "1"
+          seed: 42
+          masking:
+            - selector:
+                jsonpath: "name"
+              mask:
+                template: '{{printf "%.0f" .id_person}}{{.name}}'
+            - selector:
+                jsonpath: "name"
+              mask:
+                randomChoiceInUri: "pimo://surnameFR"
+              seed:
+                field: "name"
+            - selector:
+                jsonpath: "name2"
+              mask:
+                template: '{{printf "%.0f" .id_person}}{{.name2}}'
+            - selector:
+                jsonpath: "name2"
+              mask:
+                randomChoiceInUri: "pimo://surnameFR"
+              seed:
+                field: "name2"
+          EOF
+      - script: |-
+          echo '{"id": 123456789, "name": "", "name2": ""}' | pimo -s -1 -c masking.yml
+        assertions:
+          - result.code ShouldEqual 0
+          - result.systemoutjson.name ShouldEqual Perez
+          - result.systemoutjson.name2 ShouldEqual Perez
+          - result.systemerr ShouldBeEmpty
+      - script: |-
+          echo '{"id": 123456789, "name": "A", "name2": "B"}' | pimo -s -1 -c masking.yml
+        assertions:
+          - result.code ShouldEqual 0
+          - result.systemoutjson.name ShouldEqual Lopez
+          - result.systemoutjson.name2 ShouldEqual Legrand
+          - result.systemerr ShouldBeEmpty