Merge pull request #27 from advanced-security/testing

Testing, plus major changes to the patterns as a result of that testing

Author: aegilops

.github/scripts/validate.py

@@ -63,6 +63,8 @@ class Pattern:
 
     regex: Regex = field(default_factory=Regex)
 
+    expected: Optional[List[Dict[str, str]]] = None
+
     type: Optional[str] = None
     comments: List[str] = field(default_factory=list)
 

Pipfile

@@ -11,8 +11,9 @@ jinja2 = "*"
 
 [dev-packages]
 
+
 [requires]
-python_version = "3.8"
+python_version = "3.9"
 
 [scripts]
 main = "python ./.github/scripts/validate.py"

configs/README.md

@@ -61,7 +61,7 @@ Hardcoded JDBC / Spring datasource passwords which typically are in property fil
 <p>
 
 ```regex
-[a-zA-Z0-9!$%&*+?^_`{|}~-]+
+[^\r\n'"]{1,40}
 ```
 
 </p>
@@ -72,7 +72,7 @@ Hardcoded JDBC / Spring datasource passwords which typically are in property fil
 <p>
 
 ```regex
-[^0-9A-Za-z](spring.datasource.password|jdbc.password)(\s+|)=(\s+|)
+\b(spring\.datasource\.password|jdbc\.password)[ \t]{0,15}=[ \t]{0,15}['"]?
 ```
 
 </p>
@@ -81,7 +81,7 @@ Hardcoded JDBC / Spring datasource passwords which typically are in property fil
 <p>
 
 ```regex
-\z|[^0-9A-Za-z]|'
+\z|['"\r\n]
 ```
 
 </p>
@@ -103,7 +103,7 @@ Hardcoded JDBC / Spring datasource passwords which typically are in property fil
 <p>
 
 ```regex
-[^\s"'(${{)][a-zA-Z0-9!.,$%&*+?^_`{|}()~-]*
+[^\r\n"']+
 ```
 
 </p>
@@ -114,7 +114,7 @@ Hardcoded JDBC / Spring datasource passwords which typically are in property fil
 <p>
 
 ```regex
-[^0-9A-Za-z](SECRET_KEY)(\s+|)=(\s+|)("|')
+\bSECRET_KEY[ \t]*=[ \t]*["']
 ```
 
 </p>
@@ -123,7 +123,7 @@ Hardcoded JDBC / Spring datasource passwords which typically are in property fil
 <p>
 
 ```regex
-\z|[^a-zA-Z0-9\s!.,$%&*+?^_`{|}()~-]|'|"
+['"]
 ```
 
 </p>
@@ -140,15 +140,15 @@ Pattern to find Static passwords in YAML configuration files
 
 - The hardcoded password is between 12 and 32 chars long
 - Some false positives in Code might appear
-- The pattern only checks for cerain key words to begin the pattern (`secret:`, `password:`, etc.)
+- The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)
 
 
 <details>
 <summary>Pattern Format</summary>
 <p>
 
 ```regex
-[a-zA-Z0-9%!#$%&*+=?^_-{|}~\.,]{12,32}
+[^\r\n'"]{12,32}
 ```
 
 </p>
@@ -159,7 +159,7 @@ Pattern to find Static passwords in YAML configuration files
 <p>
 
 ```regex
-[^0-9A-Za-z](\s+|)(secret|service_pass(wd|word|code|phrase)|pass(wd|word|code|phrase)|key)(\s+|):(\s+|)
+(?:\n|\A)[ \t]{0,10}(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key)[ \t]{0,30}:[ \t]{0,30}['"]?
 ```
 
 </p>
@@ -168,8 +168,25 @@ Pattern to find Static passwords in YAML configuration files
 <p>
 
 ```regex
-[^0-9A-Za-z'"\(\)]|\z
+['"\r\n]|\z
 ```
 
+</p>
+</details>
+<details>
+<summary>Additional Matches</summary>
+<p>
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match: `^keyPassphrase$`
+- Not Match: `^.* = (?:None|True|False),?$`
+- Not Match: `^.* = \.\.\.,?$`
+- Not Match: `^(?:this\.)?[A-Za-z_]+\,$`
+- Not Match: `^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$`
+- Not Match: `^(?:str|int|bool)( +#.*)?$`
+- Not Match: `^[ \t]+$`
+- Not Match: `^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$`
+
 </p>
 </details>

configs/application.properties

@@ -11,8 +11,13 @@ spring.datasource.password = root
 # no spaces
 spring.datasource.password=SQLSpringPassword
 # quoted password
-spring.datasource.password='SQLSpringPassword'
+spring.datasource.password='QuotedSpringPassword'
 
+# Encrypted
+datasource.driver=com.mysql.jdbc.Driver 
+datasource.url=jdbc:mysql://localhost/reportsdb 
+datasource.username=reportsUser 
+datasource.password=ENC(G6N718UuyPE5bHyWKyuLQSm02auQPUtm) 
 
 
 # Sources:
@@ -21,27 +26,27 @@ spring.datasource.password='SQLSpringPassword'
 # H2 DB
 spring.datasource.url=jdbc:h2:file:C:/temp/test
 spring.datasource.username=sa
-spring.datasource.password=
+spring.datasource.password=dbpass1
 spring.datasource.driverClassName=org.h2.Driver
 spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
 
 # MySQL
 spring.datasource.url=jdbc:mysql://localhost:3306/test
 spring.datasource.username=dbuser
-spring.datasource.password=dbpass
+spring.datasource.password=dbpass2
 spring.datasource.driver-class-name=com.mysql.jdbc.Driver
 spring.jpa.database-platform=org.hibernate.dialect.MySQL5InnoDBDialect
 
 # Oracle
 spring.datasource.url=jdbc:oracle:thin:@localhost:1521:orcl
 spring.datasource.username=dbuser
-spring.datasource.password=dbpass
+spring.datasource.password=dbpass3
 spring.datasource.driver-class-name=oracle.jdbc.OracleDriver
 spring.jpa.database-platform=org.hibernate.dialect.Oracle10gDialect
 
 # SQL Server
 spring.datasource.url=jdbc:sqlserver://localhost;databaseName=springbootdb
 spring.datasource.username=dbuser
-spring.datasource.password=dbpass
+spring.datasource.password=dbpass4
 spring.datasource.driverClassName=com.microsoft.sqlserver.jdbc.SQLServerDriver
 spring.jpa.hibernate.dialect=org.hibernate.dialect.SQLServer2012Dialect

configs/global_settings.py

@@ -0,0 +1,4 @@
+# this is a fake/sample Django setting file
+
+SECRET_KEY = '!r7!(xjadix=(m5t9$0y%+bdxs#$^4u+7(s+kg&m67o0jsj&b$' # sample
+

configs/patterns.yml

@@ -25,25 +25,51 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        [a-zA-Z0-9!$%&*+?^_`{|}~-]+
+        [^\r\n'"\p{Cc}]+
       start: |
-        [^0-9A-Za-z](spring.datasource.password|jdbc.password)(\s+|)=(\s+|)
+        (?:spring\.datasource|jdbc)\.password[ \t]*=[ \t]*['"]?
       end: |
-        \z|[^0-9A-Za-z]|'
+        \z|['"\r\n]
+    expected:
+      - name: application.properties
+        start_offset: 314
+        end_offset: 318
+      - name: application.properties
+        start_offset: 358
+        end_offset: 375
+      - name: application.properties
+        start_offset: 422
+        end_offset: 442
+      - name: application.properties
+        start_offset: 836
+        end_offset: 843
+      - name: application.properties
+        start_offset: 1078
+        end_offset: 1085
+      - name: application.properties
+        start_offset: 1346
+        end_offset: 1353
+      - name: application.properties
+        start_offset: 1633
+        end_offset: 1640
+
 
   - name: Django Secret Key
     type: django_secret_key
     regex:
       version: 0.1
       pattern: |
-        [^\s"'(${{)][a-zA-Z0-9!.,$%&*+?^_`{|}()~-]*
+        [^\r\n"']+
       start: |
-        [^0-9A-Za-z](SECRET_KEY)(\s+|)=(\s+|)("|')
+        \bSECRET_KEY[ \t]*=[ \t]*["']
       end: |
-        \z|[^a-zA-Z0-9\s!.,$%&*+?^_`{|}()~-]|'|"
-
+        ['"]
     comments:
       - "_If the secret is at the start of the file, its not picked up_"
+    expected:
+      - name: global_settings.py
+        start_offset: 59
+        end_offset: 109
 
 
   # Experimental
@@ -56,13 +82,31 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        [a-zA-Z0-9%!#$%&*+=?^_-{|}~\.,]{12,32}
+        [^\r\n'"]*
       start: |
-        [^0-9A-Za-z](\s+|)(secret|service_pass(wd|word|code|phrase)|pass(wd|word|code|phrase)|key)(\s+|):(\s+|)
+        (?:\n|\A)[ \t]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key)[ \t]*:[ \t]*['"]?
       end: |
-        [^0-9A-Za-z'"\(\)]|\z
-
+        ['"\r\n]|\z
+      additional_not_match:
+        - ^keyPassphrase$
+        - ^.* = (?:None|True|False),?$
+        - ^.* = \.\.\.,?$
+        - ^(?:(?:this|self|obj)\.)?[A-Za-z_]+\,$
+        - ^(?:(?:this|self|obj)\.)[A-Za-z_].*$
+        - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
+        - ^(?:str|int|bool)( +#.*)?$
+        - ^[ \t]+$
+        - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+        - ^\$\{[A-Za-z0-9_-]+\}$
     comments:
       - "The hardcoded password is between 12 and 32 chars long"
       - "Some false positives in Code might appear"
-      - "The pattern only checks for cerain key words to begin the pattern (`secret:`, `password:`, etc.)"
+      - "The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)"
+    expected:
+      - name: example.yml
+        start_offset: 57
+        end_offset: 80
+      - name: example.yml
+        start_offset: 57
+        end_offset: 80
+

generic/README.md

@@ -21,7 +21,7 @@
 <p>
 
 ```regex
-[^\t "'(${{)][a-zA-Z0-9\t !.,$%&*+?^_`{|}()~-]+
+[a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
 ```
 
 </p>
@@ -32,7 +32,7 @@
 <p>
 
 ```regex
-(?i)((api|jwt|mysql|)?(_|-|.)?((pass|pas)(wd|wrd|word|code|phrase)|pass|pwd|secret|token))([\t ]+|)(=|:)([\t ]+|)("|'|[\t ]|)
+(?i)(?:api|jwt|mysql)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|pwd|secret)[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
 ```
 
 </p>
@@ -41,7 +41,7 @@
 <p>
 
 ```regex
-\z|[^a-zA-Z0-9\t !.,$%&*+?^_`{|}()~-]|'|"
+(\z|[\r\n'"])
 ```
 
 </p>
@@ -52,7 +52,43 @@
 Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
 
 
-- Not Match: `\b((?i)(pass|pas)(wd|wrd|word|code|phrase)|pass|pwd|secret|token|write|read|on|off|true|false|placeholder|dummy)\b`
+- Not Match: `^(?i)(?:[A-Za-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:user|key)_?)?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase),?[\]>)]?\\?$`
+- Not Match: `^.*token.*$`
+- Not Match: `^[a-zA-Z0-9._]+[_.](?:password|passphrase|secret|key).*$`
+- Not Match: `^.* passphrase .*$`
+- Not Match: `^(?i)(?:[a-zA-Z0-9_.]*,\s*)?[[<(]?(?:write|read|on|off|true|false|none|null|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?$`
+- Not Match: `^\s*%[sr]\s*$`
+- Not Match: `^\s*$`
+- Not Match: `^\s*(?:int|str|(?:typing\.)?Any|None|bytes|bool|ReadableBuffer)\s*([,|].*)?\s*$`
+- Not Match: `^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$`
+- Not Match: `^\s*\.\.\.,?\s*$`
+- Not Match: `^\s*\\\s*$`
+- Not Match: `^\\n$`
+- Not Match: `^\s*,s*$`
+- Not Match: `^\\0$`
+- Not Match: `^function\s*\([^)]*\)\s*{\s*`
+- Not Match: `^\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])$`
+- Not Match: `^\s*[0-9]{1,4}(?:\s*(?:/\*|#|//).*)?$`
+- Not Match: `^(?:new )?[a-zA-Z0-9_.]+\(.*$`
+- Not Match: `^\s*(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_]+[,[]?\s*$`
+- Not Match: `^\s*[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?\s*$`
+- Not Match: `^\s*(?:~|/tmp|\.\.|\.)\s*$`
+- Not Match: `^\\{1,2}w\+/g,( \\?)?$`
+- Not Match: `^\s*\$\{[^}]+\}\s*$`
+- Not Match: `^\s*\$\([^)]+\)\s*$`
+- Not Match: `^\s*\{[^}]*\}\s*$`
+- Not Match: `^\s*\[[^\]]*\]\s*$`
+- Not Match: `^[,()[\]{}`.]\\?$`
+- Not Match: `^geheim\$parole$`
+- Not Match: `^\s*\([Oo]ptional\).*$`
+- Not Match: `^-[)(]$`
+- Not Match: `^0x[A-Fa-f0-9]+,?$`
+- Not Match: `^\$[1-9]$`
+- Not Match: `^\$[A-Za-z0-9_]+$`
+- Not Match: `^[0-9],?$`
+- Not Match: `^\s*ALL(?:\\n)?\s*$`
+- Not Match: `^(?:public|private) [A-Za-z0-9_]+ \{$`
+- Not Match: `^\$[a-zA-Z0-9_]+\{$`
 
 </p>
 </details>
@@ -70,9 +106,41 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 <p>
 
 ```regex
-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
+(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
 ```
 
 </p>
 </details>
 
+<details>
+<summary>Start Pattern</summary>
+<p>
+
+```regex
+\A|[^0-9A-Fa-f-]
+```
+
+</p>
+</details><details>
+<summary>End Pattern</summary>
+<p>
+
+```regex
+\z|[^0-9A-Fa-f-]
+```
+
+</p>
+</details>
+<details>
+<summary>Additional Matches</summary>
+<p>
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match: `^12345678-1234-5678-1234-567812345678$`
+- Not Match: `^00000000-0000-0000-0000-000000000000$`
+- Not Match: `^(?i)00010203-0405-0607-0809-0a0b0c0d0e0f$`
+- Not Match: `^(?i)12345678-1234-1234-1234-123456789abc$`
+
+</p>
+</details>