Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix url arguments detection #494

Merged
merged 6 commits into from
Jan 24, 2024
Merged

Fix url arguments detection #494

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bebf546
fix detection of url argument variables
babenek Jan 23, 2024
135d466
skip / from variable name
babenek Jan 23, 2024
8c15f44
after correction
babenek Jan 23, 2024
e693fe8
BM scores upd
babenek Jan 23, 2024
6d80108
BM scores fix
babenek Jan 23, 2024
6cffb1a
Apply suggestions from code review
babenek Jan 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 29 additions & 29 deletions cicd/benchmark.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
DATA: 19360133 valid lines. MARKUP: 73578 items
DATA: 19355695 valid lines. MARKUP: 73551 items
Category Positives Negatives Template
-------------------------- ----------- ----------- ----------
Authentication Key & Token 91 2 31
Generic Secret 1063 16 204
Generic Token 331 46 558
Other 829 62708 610
Password 1396 117 4092
Predefined Pattern 373 3 11
Private Key 1012 29
Seed, Salt, Nonce 48 4 4
TOTAL: 5143 62925 5510
Generic Secret 1063 17 207
Generic Token 328 45 555
Other 822 62688 610
Password 1398 117 4092
Predefined Pattern 376 3 11
Private Key 1011 30
Seed, Salt, Nonce 51 2 1
TOTAL: 5140 62904 5507
FileType FileNumber ValidLines Positives Negatives Template
--------------- ------------ ------------ ----------- ----------- ----------
224 41595 44 845 80
Expand Down Expand Up @@ -81,7 +81,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.haml 13 305 19
.hbs 4 108 7
.hs 19 5127 31 53 6
.html 66 32371 9 167 18
.html 65 32317 9 165 18
.idl 2 841 6
.iml 6 699 31
.in 6 2190 2 41 7
Expand All @@ -91,11 +91,11 @@ FileType FileNumber ValidLines Positives Negatives Templat
.ipynb 1 210 4
.j 1 329 2
.j2 32 6327 8 183 11
.java 626 178326 165 1422 177
.java 626 178326 171 1419 177
.jenkinsfile 1 78 1 6
.jinja2 4 146 7
.js 766 736851 316 3096 365
.json 930 15031039 281 13783 186
.js 762 736564 313 3092 365
.json 929 15031027 279 13784 186
.jsp 13 4101 1 38 1
.jsx 7 1162 19
.jwt 6 8 6
Expand All @@ -122,7 +122,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.markdown 3 146 2 2
.markerb 3 12 2 1
.marko 1 32 2
.md 718 183585 371 2888 728
.md 716 180742 367 2881 725
.mdx 3 723 7
.mf 1 6 2
.mjml 2 183 3
Expand Down Expand Up @@ -223,7 +223,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.toml 83 2566 13 85 142
.tpl 1 50 1
.travis 1 34 2 3 1
.ts 675 151241 88 2185 239
.ts 674 149999 88 2179 239
.tsx 57 13349 1 121 5
.ttar 2 6526 8 3
.txt 450 84353 1695 9178 91
Expand All @@ -241,17 +241,17 @@ FileType FileNumber ValidLines Positives Negatives Templat
.yml 480 43349 283 1344 360
.zsh 8 1779 14
.zsh-theme 7 285 8
TOTAL: 11250 19360133 5143 62925 5510
Detected Credentials: 6112
credsweeper result_cnt : 5230, lost_cnt : 0, true_cnt : 4349, false_cnt : 881
Category TP FP TN FN FPR FNR ACC PRC RCL F1
-------------------------- ---- ---- -------- ---- --------- ---------- -------- -------- -------- --------
Authentication Key & Token 76 5 28 15 0.151515 0.164835 0.83871 0.938272 0.835165 0.883721
Generic Secret 979 7 213 84 0.0318182 0.0790216 0.929072 0.992901 0.920978 0.955588
Generic Token 295 12 592 36 0.0198676 0.108761 0.948663 0.960912 0.891239 0.924765
Other 581 722 62596 248 0.0114028 0.299156 0.984878 0.445894 0.700844 0.545028
Password 1006 131 4078 390 0.0311238 0.27937 0.907047 0.884785 0.72063 0.794315
Predefined Pattern 356 2 12 17 0.142857 0.0455764 0.950904 0.994413 0.954424 0.974008
Private Key 1011 0 29 1 0.00098814 0.999039 1 0.999012 0.999506
Seed, Salt, Nonce 45 2 6 3 0.25 0.0625 0.910714 0.957447 0.9375 0.947368
4349 881 19354109 794 4.552e-05 0.154385 0.999913 0.831549 0.845615 0.838523
TOTAL: 11241 19355695 5140 62904 5507
Detected Credentials: 6103
credsweeper result_cnt : 5222, lost_cnt : 0, true_cnt : 4347, false_cnt : 875
Category TP FP TN FN FPR FNR ACC PRC RCL F1
-------------------------- ---- ---- -------- ---- --------- --------- -------- -------- -------- --------
Authentication Key & Token 76 5 28 15 0.151515 0.164835 0.83871 0.938272 0.835165 0.883721
Generic Secret 979 7 217 84 0.03125 0.0790216 0.929293 0.992901 0.920978 0.955588
Generic Token 292 9 591 36 0.015 0.109756 0.951509 0.9701 0.890244 0.928458
Other 574 721 62577 248 0.0113906 0.301703 0.984888 0.443243 0.698297 0.542277
Password 1009 131 4078 389 0.0311238 0.278255 0.907259 0.885088 0.721745 0.795114
Predefined Pattern 359 2 12 17 0.142857 0.0452128 0.951282 0.99446 0.954787 0.97422
Private Key 1011 0 30 0 1 1 1 1
Seed, Salt, Nonce 47 0 3 4 0.0784314 0.925926 1 0.921569 0.959184
4347 875 19349680 793 4.522e-05 0.15428 0.999914 0.83244 0.84572 0.839027
6 changes: 3 additions & 3 deletions credsweeper/common/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@

class KeywordPattern:
"""Pattern set of keyword types"""
key_left = r"(?P<variable>(([`'\"]+[^:='\"`<>&]*|[^:='\"`<>\s\(&]*)" \
key_left = r"(?P<variable>(([`'\"]+[^:='\"`}<>\\/&?]*|[^:='\"`}<>\s()\\/&?]*)" \
r"(?P<keyword>"
# there will be inserted a keyword
key_right = r")" \
r"[^:='\"`<>\?\!]*)[`'\"]*)" # <variable>
r"[^:='\"`<>{?!&]*)[`'\"]*)" # <variable>
separator = r"\s*\]?\s*" \
r"(?P<separator>:( [a-z]{3,9} )?=|:|=>|!=|==|=)" \
r"(?P<separator>:( [a-z]{3,9} )?=|:|=>|!=|===|==|=)" \
r"((?!\s*ENC(\(|\[))(\s|\w)*\((\s|\w|=|\()*|\s*)"
value = r"(?P<value_leftquote>((b|r|br|rb|u|f|rf|fr|\\)?[`'\"])+)?" \
r"(?P<value>(?:\{[^}]{3,8000}\})|(?:<[^>]{3,8000}>)|" \
Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Common requirements
beautifulsoup4==4.12.2
cryptography==41.0.7
beautifulsoup4==4.12.3
cryptography==42.0.0
GitPython==3.1.41
google-auth-oauthlib==1.2.0
humanfriendly==10.0
Expand Down
6 changes: 3 additions & 3 deletions tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
SAMPLES_FILES_COUNT: int = 124

# credentials count after scan
SAMPLES_CRED_COUNT: int = 394
SAMPLES_CRED_LINE_COUNT: int = 411
SAMPLES_CRED_COUNT: int = 395
SAMPLES_CRED_LINE_COUNT: int = 412

# credentials count after post-processing
SAMPLES_POST_CRED_COUNT: int = 312
SAMPLES_POST_CRED_COUNT: int = 313

# with option --doc
SAMPLES_IN_DOC = 396
Expand Down
58 changes: 41 additions & 17 deletions tests/data/depth_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@
"value": "JgEWaL6V6eM%2FFb9wuXG4I3IB6wY%3D",
"value_start": 44,
"value_end": 76,
"variable": "/oauth_signature",
"variable": "oauth_signature",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 4.288909765557392,
Expand Down Expand Up @@ -1328,7 +1328,7 @@
"value": "dipPr125Gg!",
"value_start": 25,
"value_end": 36,
"variable": "master/Password",
"variable": "Password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.1449378351248165,
Expand Down Expand Up @@ -1616,7 +1616,7 @@
"value": "dipPr141Gg!",
"value_start": 15,
"value_end": 26,
"variable": "master/PASS",
"variable": "PASS",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 2.9631196533066344,
Expand Down Expand Up @@ -1913,6 +1913,30 @@
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "VALIDATED_KEY",
"ml_probability": 0.75145,
"rule": "Password",
"severity": "medium",
"line_data_list": [
{
"line": "account:dipPr155Gg!/password:dipPr155Gg!",
"line_num": 56,
"path": "tests/samples/doc_id_pair_passwd_pair",
"info": "tests/samples/doc_id_pair_passwd_pair|RAW",
"value": "dipPr155Gg!",
"value_start": 29,
"value_end": 40,
"variable": "password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 2.9631196533066344,
"valid": false
}
}
]
},
{
"api_validation": "NOT_AVAILABLE",
"ml_validation": "VALIDATED_KEY",
Expand Down Expand Up @@ -2264,7 +2288,7 @@
"value": "dipPr175Gg!",
"value_start": 26,
"value_end": 37,
"variable": "master/Password",
"variable": "Password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.1449378351248165,
Expand Down Expand Up @@ -2336,7 +2360,7 @@
"value": "dipPr178Gg!",
"value_start": 19,
"value_end": 30,
"variable": "master/Password",
"variable": "Password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.1449378351248165,
Expand Down Expand Up @@ -2672,7 +2696,7 @@
"value": "master/iPp2@GRq",
"value_start": 12,
"value_end": 27,
"variable": "ID/Password",
"variable": "Password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.64643122256795,
Expand All @@ -2696,7 +2720,7 @@
"value": "master/iPp3@GRq",
"value_start": 8,
"value_end": 23,
"variable": "ID/Pass",
"variable": "Pass",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.64643122256795,
Expand Down Expand Up @@ -2744,7 +2768,7 @@
"value": "master/iPp7@GRq",
"value_start": 18,
"value_end": 33,
"variable": "username/password",
"variable": "password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.64643122256795,
Expand All @@ -2768,7 +2792,7 @@
"value": "master/iPp8@GRq",
"value_start": 10,
"value_end": 25,
"variable": "id/passwd",
"variable": "passwd",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.64643122256795,
Expand Down Expand Up @@ -2840,7 +2864,7 @@
"value": "master/iPp19@GRq",
"value_start": 18,
"value_end": 34,
"variable": "id/pw",
"variable": "pw",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.75,
Expand All @@ -2864,7 +2888,7 @@
"value": "master,iPp20@GRq",
"value_start": 8,
"value_end": 24,
"variable": "id/pass",
"variable": "pass",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.5,
Expand All @@ -2888,7 +2912,7 @@
"value": "master/iPp21@GRq",
"value_start": 7,
"value_end": 23,
"variable": "ID/PWD",
"variable": "PWD",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.75,
Expand All @@ -2912,7 +2936,7 @@
"value": "master/iPp25@GRq",
"value_start": 10,
"value_end": 26,
"variable": "user/pass",
"variable": "pass",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.75,
Expand All @@ -2936,7 +2960,7 @@
"value": "master/iPp27@GRq",
"value_start": 12,
"value_end": 28,
"variable": "ID/Password",
"variable": "Password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.75,
Expand Down Expand Up @@ -5168,7 +5192,7 @@
"value": "IhqSb1Gg",
"value_start": 44,
"value_end": 52,
"variable": "{1}/password",
"variable": "password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.0,
Expand Down Expand Up @@ -5216,7 +5240,7 @@
"value": "IhqSb1Gg",
"value_start": 49,
"value_end": 57,
"variable": "{1}/password",
"variable": "password",
"entropy_validation": {
"iterator": "BASE64_CHARS",
"entropy": 3.0,
Expand Down Expand Up @@ -5408,7 +5432,7 @@
"value": "gireogicracklecrackle1231567190113413981",
"value_start": 15,
"value_end": 55,
"variable": "\\ngit_token",
"variable": "ngit_token",
"entropy_validation": {
"iterator": "BASE36_CHARS",
"entropy": 3.97402442086502,
Expand Down
Loading