feat: improve testing for rules (#1404)

* feat: improve testing for rules

* chore: add expected to jsonv2

* chore: add test for expected rules in JSONV2

* refactor: make it less costly to find expected rules

e2e/rules/.snapshots/TestExpectedRule-testdata-data-expected_rule

@@ -0,0 +1,5 @@
+{"source":"Bearer","version":"dev","findings":[{"cwe_ids":["319"],"id":"expected_rule","title":"","description":"","documentation_url":"","line_number":3,"full_filename":"e2e/rules/testdata/data/expected_rule/main.rb","filename":"main.rb","source":{"start":3,"end":3,"column":{"start":3,"end":7}},"sink":{"start":3,"end":3,"column":{"start":3,"end":7},"content":"sink"},"parent_line_number":3,"snippet":"sink","fingerprint":"c50ecec7e1fcfba6cce5fcfab129556c_0","old_fingerprint":"6630ae26e5210b1e43bb4c02426e6be7_0","code_extract":"  sink","severity":"low"},{"cwe_ids":["319"],"id":"expected_rule","title":"","description":"","documentation_url":"","line_number":8,"full_filename":"e2e/rules/testdata/data/expected_rule/main.rb","filename":"main.rb","source":{"start":8,"end":8,"column":{"start":3,"end":7}},"sink":{"start":8,"end":8,"column":{"start":3,"end":7},"content":"sink"},"parent_line_number":8,"snippet":"sink","fingerprint":"c50ecec7e1fcfba6cce5fcfab129556c_1","old_fingerprint":"6630ae26e5210b1e43bb4c02426e6be7_1","code_extract":"  sink","severity":"low"}],"expected_findings":[{"rule_id":"expected_rule","location":{"start":3,"end":3,"column":{"start":3,"end":7}}},{"rule_id":"expected_rule","location":{"start":8,"end":8,"column":{"start":3,"end":7}}}]}
+
+--
+Analyzing codebase
+

e2e/rules/helper.go

@@ -24,6 +24,22 @@ func buildRulesTestCase(testName, path, ruleID string) testhelper.TestCase {
 	return testhelper.NewTestCase(testName, arguments, options)
 }
 
+func buildRulesTestCaseJsonV2(testName, path, ruleID string) testhelper.TestCase {
+	arguments := []string{
+		"scan",
+		path,
+		"--only-rule=" + ruleID,
+		"--format=jsonv2",
+		"--disable-default-rules",
+		"--exit-code=0",
+		"--external-rule-dir=" + filepath.Join("e2e", "rules", "testdata", "rules"),
+	}
+
+	options := testhelper.TestCaseOptions{}
+
+	return testhelper.NewTestCase(testName, arguments, options)
+}
+
 func runRulesTest(folderPath string, ruleID string, t *testing.T) {
 	snapshotDirectory := ".snapshots"
 

e2e/rules/rules_test.go

@@ -55,3 +55,18 @@ func TestRubyRailsDefaultEncryptionStructure(t *testing.T) {
 func TestRubyRailsDefaultEncryptionSchema(t *testing.T) {
 	runRulesTest("ruby_rails_default_encryption_schema_rb", "ruby_rails_default_encryption", t)
 }
+
+func TestExpectedRule(t *testing.T) {
+	testDataDir := "testdata/data/expected_rule"
+
+	testCases := []testhelper.TestCase{}
+	testCases = append(testCases,
+		buildRulesTestCaseJsonV2(
+			testDataDir,
+			filepath.Join("e2e", "rules", testDataDir),
+			"expected_rule",
+		),
+	)
+
+	testhelper.RunTestsWithSnapshotSubdirectory(t, testCases, ".snapshots")
+}

e2e/rules/testdata/data/expected_rule/main.rb

@@ -0,0 +1,14 @@
+def m
+  # bearer:expected expected_rule
+  sink
+end
+
+def n
+  # bearer:expected expected_rule
+  sink
+end
+
+def foo
+  bar
+end
+

e2e/rules/testdata/rules/expected_rule.yml

@@ -0,0 +1,9 @@
+patterns:
+  - sink
+languages:
+  - ruby
+severity: low
+metadata:
+  cwe_id:
+    - 319
+  id: expected_rule