fix: performance on large repositories (#220)

* wip: ignore symlink and extend to bigger files * wip: improve processor efficiency * refactor: improve performance of processors * refactor: cleanup * fix: use correct encrypted field module name and update snapshots Co-authored-by: David Roe <[email protected]>
Bearer · Dec 13, 2022 · 480b746 · 480b746
1 parent cf89a53
commit 480b746
Show file tree

Hide file tree

Showing 45 changed files with 775 additions and 252 deletions.
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_cookies b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_cookies
@@ -44,4 +44,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_jwt b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_jwt
@@ -34,4 +34,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_session b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_session
@@ -26,4 +26,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger b/integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger
@@ -44,4 +44,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-ftp b/integration/custom_detectors/.snapshots/TestCustomDetectors-ftp
@@ -85,4 +85,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection b/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection
@@ -173,4 +173,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_http_detection b/integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_http_detection
@@ -475,4 +475,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/...ion/custom_detectors/.snapshots/TestCustomDetectors-ssl_certificate_verification_disabled b/...ion/custom_detectors/.snapshots/TestCustomDetectors-ssl_certificate_verification_disabled
@@ -30,4 +30,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/flags/.snapshots/TestInitCommand-init b/integration/flags/.snapshots/TestInitCommand-init
@@ -14,7 +14,6 @@ scan:
             languages:
                 - ruby
             param_parenting: true
-            processors: []
             patterns:
                 - pattern: |
                     class $CLASS_NAME < ApplicationRecord
@@ -33,7 +32,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     cookies[...] = $ANYTHING
@@ -70,7 +68,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Rails.application.configure do
@@ -89,7 +86,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Net::FTP.new()
@@ -109,7 +105,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Net::FTP.open do
@@ -128,7 +123,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Rails.application.configure do
@@ -156,7 +150,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     JWT.encode(<$ARGUMENT>)
@@ -173,7 +166,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     session[...] = $ANYTHING
@@ -190,7 +182,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     logger.info(<$ARGUMENT>)
@@ -210,7 +201,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Sentry::Breadcrumb.new(<$ARGUMENT>)
@@ -238,47 +228,6 @@ scan:
             languages:
                 - sql
             param_parenting: true
-            processors:
-                - query: |
-                    verified_by = data.bearer.encrypted_verified.verified_by
-                    encrypted = data.bearer.encrypted_verified.encrypted
-                  modules:
-                    - name: bearer.encrypted_verified
-                      content: |-
-                        package bearer.encrypted_verified
-
-                        import future.keywords
-
-
-                        default encrypted := false
-
-
-                        ruby_encrypted[location] {
-                            some detection in input.all_detections
-                            detection.detector_type == "detect_encrypted_ruby_class_properties"
-                            detection.value.classification.decision.state == "valid"
-                            location = detection
-                        }
-
-                        encrypted = true {
-                            some detection in ruby_encrypted
-                            detection.value.object_name == input.target.value.object_name
-                            detection.value.field_name == input.target.value.field_name
-                            input.target.value.field_name != ""
-                            input.target.value.object_name != ""
-                        }
-
-                        verified_by[verification] {
-                            some detection in ruby_encrypted
-                            detection.value.object_name == input.target.value.object_name
-                            detection.value.field_name == input.target.value.field_name
-
-                            verification = {
-                                "detector": "detect_encrypted_ruby_class_properties",
-                                "filename": detection.source.filename,
-                                "line_number": detection.source.line_number
-                            }
-                        }
             patterns:
                 - pattern: |
                     CREATE TABLE public.$TABLE_NAME (
@@ -302,7 +251,6 @@ scan:
             languages:
                 - ruby
             param_parenting: true
-            processors: []
             patterns:
                 - pattern: |
                     CSV.generate { <$DATA_TYPE> }
@@ -333,7 +281,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     URI.encode_www_form(<$DATA_TYPE>)
@@ -366,7 +313,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     URI(<$INSECURE_URL>)
@@ -396,7 +342,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     URI.encode_www_form(<$DATA_TYPE>)
@@ -429,7 +374,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Net::HTTP.post_form(<$INSECURE_URL>)
@@ -459,7 +403,6 @@ scan:
             languages:
                 - ruby
             param_parenting: false
-            processors: []
             patterns:
                 - pattern: |
                     Net::HTTP.start($_, $_, $_, :verify_mode => OpenSSL::SSL::VERIFY_NONE) do
@@ -1522,7 +1465,7 @@ scan:
     skip-path: []
 worker:
     existing-worker: ""
-    file-size-max: 100000
+    file-size-max: 2000000
     files-to-batch: 1
     memory-max: 800000000
     timeout: 10m0s

diff --git a/integration/flags/.snapshots/TestMetadataFlags-help-scan b/integration/flags/.snapshots/TestMetadataFlags-help-scan
@@ -24,7 +24,7 @@ Policy Flags
 
 Worker Flags
       --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 100000)
+      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
       --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
       --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
       --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)

diff --git a/integration/flags/.snapshots/TestMetadataFlags-scan-help b/integration/flags/.snapshots/TestMetadataFlags-scan-help
@@ -24,7 +24,7 @@ Policy Flags
 
 Worker Flags
       --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 100000)
+      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
       --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
       --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
       --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)

diff --git a/integration/flags/.snapshots/TestReportFlags-domain-resolution-disabled b/integration/flags/.snapshots/TestReportFlags-domain-resolution-disabled
@@ -1,4 +1,6 @@
 [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
+Processing Detectors
+Finished processing Detectors
 
diff --git a/integration/flags/.snapshots/TestReportFlags-format-json b/integration/flags/.snapshots/TestReportFlags-format-json
@@ -1,4 +1,6 @@
 [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
+Processing Detectors
+Finished processing Detectors
 
diff --git a/integration/flags/.snapshots/TestReportFlags-format-yaml b/integration/flags/.snapshots/TestReportFlags-format-yaml
@@ -64,4 +64,6 @@
 
 
 --
+Processing Detectors
+Finished processing Detectors
 
diff --git a/integration/flags/.snapshots/TestReportFlags-health-context b/integration/flags/.snapshots/TestReportFlags-health-context
@@ -1,4 +1,6 @@
 [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
+Processing Detectors
+Finished processing Detectors
 
diff --git a/integration/flags/.snapshots/TestReportFlags-report-dataflow b/integration/flags/.snapshots/TestReportFlags-report-dataflow
@@ -1,4 +1,8 @@
 {"data_types":[{"name":"Email Address","detectors":[{"name":"ruby","locations":[{"filename":"testdata/simple/main.rb","line_number":1}]}]}],"risks":[{"detector_id":"detect_ruby_logger","data_types":[{"name":"Email Address","stored":false,"locations":[{"filename":"testdata/simple/main.rb","line_number":1,"parent":{"line_number":1,"content":"logger.info(\"user info\", user.email)"}}]}]}],"components":[]}
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/flags/.snapshots/TestReportFlags-report-dataflow-verified-by b/integration/flags/.snapshots/TestReportFlags-report-dataflow-verified-by
@@ -144,4 +144,8 @@ components: []
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
 
diff --git a/integration/flags/.snapshots/TestReportFlags-report-detectors b/integration/flags/.snapshots/TestReportFlags-report-detectors
@@ -1,4 +1,6 @@
 [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
+Processing Detectors
+Finished processing Detectors
 
diff --git a/integration/flags/.snapshots/TestReportFlags-report-policies b/integration/flags/.snapshots/TestReportFlags-report-policies
@@ -11,4 +11,38 @@ high:
 
 
 --
+Processing Detectors
+Finished processing Detectors
+Processing Dataflow
+Finished processing Dataflow
+Processing Policies
+Processing policy Application level encryption missing
+Finished processing policy Application level encryption missing
+Processing policy Cookie leaking
+Finished processing policy Cookie leaking
+Processing policy HTTP GET parameters
+Finished processing policy HTTP GET parameters
+Processing policy Insecure communication
+Finished processing policy Insecure communication
+Processing policy Insecure FTP
+Finished processing policy Insecure FTP
+Processing policy Insecure FTP with Data Category
+Finished processing policy Insecure FTP with Data Category
+Processing policy Insecure HTTP GET
+Finished processing policy Insecure HTTP GET
+Processing policy Insecure HTTP with Data Category
+Finished processing policy Insecure HTTP with Data Category
+Processing policy Insecure SMTP
+Finished processing policy Insecure SMTP
+Processing policy JWT leaking
+Finished processing policy JWT leaking
+Processing policy Logger leaking
+Finished processing policy Logger leaking
+Processing policy Third-party data category exposure
+Finished processing policy Third-party data category exposure
+Processing policy Session leaking
+Finished processing policy Session leaking
+Processing policy SSL certificate verification disabled
+Finished processing policy SSL certificate verification disabled
+Finished processing policies
 
diff --git a/integration/flags/.snapshots/TestReportFlags-skipped-paths b/integration/flags/.snapshots/TestReportFlags-skipped-paths
@@ -1,4 +1,6 @@
 [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
 
 --
+Processing Detectors
+Finished processing Detectors
Original file line number	Diff line number	Diff line change
Expand Up		@@ -64,4 +64,6 @@


		--
		Processing Detectors
		Finished processing Detectors