diff --git a/download_data.py b/download_data.py index 2257b25c5..727932a47 100644 --- a/download_data.py +++ b/download_data.py @@ -357,9 +357,12 @@ def generate_value(value): base_32 = True hex_upper = True hex_lower = True - for i in value: + for n, i in enumerate(value): if base_32 and i not in "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567": base_32 = False + if '-' == i and 36 == len(value) and n in [8, 13, 18, 23]: + # UUID separator + continue if hex_upper and i not in "0123456789ABCDEF": hex_upper = False if hex_lower and i not in "0123456789abcdef": diff --git a/meta/31423103.csv b/meta/31423103.csv index 6cb3254d7..eeb3d5811 100644 --- a/meta/31423103.csv +++ b/meta/31423103.csv @@ -110,3 +110,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 133260,f63d3cf7,GitHub,31423103,data/31423103/src/f63d3cf7.js,784,784,F,F,,,F,F,,,,,0,0,F,F,F,Token 133261,fa60852f,GitHub,31423103,data/31423103/src/fa60852f.js,688,688,F,F,,,F,F,,,,,0,0,F,F,F,Token 1023934,a3046da0,GitHub,31423103,data/31423103/test/a3046da0.example,46,46,F,F,,,F,F,,,,,0,0,F,F,F,Key +1338567,a3046da0,GitHub,31423103,data/31423103/test/a3046da0.example,47,47,F,F,12,44,F,F,,,,,0.0,0,F,F,F,Key diff --git a/meta/48fd3902.csv b/meta/48fd3902.csv index 4d940c2d9..72f82f937 100644 --- a/meta/48fd3902.csv +++ b/meta/48fd3902.csv @@ -223,3 +223,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1123096,238cc5a0,GitHub,48fd3902,data/48fd3902/other/238cc5a0.md,64,64,F,F,,,F,F,,,,,0.0,0,F,F,F,Password 1086791,c370d8f8,GitHub,48fd3902,data/48fd3902/test/c370d8f8.kt,1171,1171,Template,F,41,74,F,F,,,,,2.74,13,F,F,F,Password 1338373,8605db08,GitHub,48fd3902,data/48fd3902/test/8605db08.kt,58,58,T,F,43,1051,F,F,,,,,0.0,0,F,F,F,JSON Web Token +1338577,8605db08,GitHub,48fd3902,data/48fd3902/test/8605db08.kt,30,30,T,F,34,,F,F,,,,,0.0,0,F,F,F,Token diff --git a/meta/4dccc5be.csv b/meta/4dccc5be.csv index 2e88c945c..382e34d53 100644 --- a/meta/4dccc5be.csv +++ b/meta/4dccc5be.csv @@ -121,7 +121,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34014,438f1f52,GitHub,4dccc5be,data/4dccc5be/src/438f1f52.rb,46,46,Template,T,19,35,F,F,Any,,,Secret,3.38,16,F,F,F,Password 34015,438f1f52,GitHub,4dccc5be,data/4dccc5be/src/438f1f52.rb,6,6,Template,T,19,29,F,F,CharsOnly,,,Secret,3.12,10,F,F,F,Password 34141,7a820c92,GitHub,4dccc5be,data/4dccc5be/src/7a820c92.rb,12,12,Template,T,21,25,T,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password:URL Credentials -34167,b9311aeb,GitHub,4dccc5be,data/4dccc5be/other/b9311aeb,12,12,Template,T,25,29,T,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password:URL Credentials +34167,b9311aeb,GitHub,4dccc5be,data/4dccc5be/other/b9311aeb,12,12,F,T,,,T,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password:URL Credentials 34245,b1271e78,GitHub,4dccc5be,data/4dccc5be/other/b1271e78,10,10,Template,T,19,28,F,F,Any,,,Token,2.60,9,F,F,F,Token 34769,3b048855,GitHub,4dccc5be,data/4dccc5be/src/3b048855.rb,20,20,T,T,17,24,F,F,CharsOnly,,,Secret,2.52,7,F,F,F,Password 35002,8349f09f,GitHub,4dccc5be,data/4dccc5be/src/8349f09f.rb,189,189,T,T,24,39,F,F,Any,,,Secret,3.11,15,F,F,F,Password diff --git a/meta/533c47c6.csv b/meta/533c47c6.csv index b22f2da83..d370a2b8f 100644 --- a/meta/533c47c6.csv +++ b/meta/533c47c6.csv @@ -159,3 +159,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 133364,ff9bd4ae,GitHub,533c47c6,data/533c47c6/test/ff9bd4ae.py,124,124,F,F,,,F,F,,,,,0,0,F,F,F,API 100002,a99307c6,GitHub,533c47c6,data/533c47c6/test/a99307c6.py,171,171,F,T,32,40,F,F,CharsOnly,,,Secret,3.00,8,F,F,F,Password 100003,a99307c6,GitHub,533c47c6,data/533c47c6/test/a99307c6.py,176,176,F,T,32,40,F,F,CharsOnly,,,Secret,3.00,8,F,F,F,Password +1338576,5af8e7ca,GitHub,533c47c6,data/533c47c6/src/5af8e7ca.py,309,309,F,F,,,F,F,,,,,0.0,0,F,F,F,Key diff --git a/meta/6c73b80a.csv b/meta/6c73b80a.csv index 0b0a53376..e7c4ab571 100644 --- a/meta/6c73b80a.csv +++ b/meta/6c73b80a.csv @@ -285,7 +285,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 34203,f1bada03,GitHub,6c73b80a,data/6c73b80a/test/f1bada03.go,3957,3957,Template,F,29,32,F,F,Any,,,Secret,1.58,3,F,F,F,Password 34204,7708ebf0,GitHub,6c73b80a,data/6c73b80a/test/7708ebf0.go,5079,5079,T,T,29,39,F,F,Any,,,Secret,2.46,10,F,F,F,Password 34205,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,4486,4486,Template,F,31,34,F,F,Any,,,Secret,1.58,3,F,F,F,Password -34230,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,707,707,T,T,35,39,T,F,CharsOnly,,,Secret,2.00,4,F,F,F,Password +34230,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,707,707,T,T,35,39,T,F,CharsOnly,,,Secret,2.00,4,F,F,F,URL Credentials 34429,633f9ac2,GitHub,6c73b80a,data/6c73b80a/test/633f9ac2.go,1136,1136,T,F,11,18,F,F,Any,,,Secret,2.12,7,F,F,F,Password 34440,728925ac,GitHub,6c73b80a,data/6c73b80a/test/728925ac.go,2386,2386,T,F,11,18,F,F,Any,,,Secret,2.12,7,F,F,F,Password 34563,924a0284,GitHub,6c73b80a,data/6c73b80a/test/924a0284.conf,13,13,T,F,10,20,F,F,Any,,,Secret,2.59,10,F,F,F,Password @@ -345,8 +345,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 35494,b406db25,GitHub,6c73b80a,data/6c73b80a/src/b406db25.pem,1,28,T,T,,,F,F,Any,Private,,Unknown,2.39,27,F,F,F,PEM Private Key 35556,81102334,GitHub,6c73b80a,data/6c73b80a/src/81102334.pem,1,28,T,T,,,F,F,Any,Private,,Unknown,2.39,27,F,F,F,PEM Private Key 35746,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2598,2598,T,T,25,29,T,F,CharsOnly,,,Secret,1.50,4,F,F,F,URL Credentials -35747,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2590,2590,T,F,25,29,T,F,CharsOnly,,,Secret,2.00,4,F,F,F,Password -35748,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,3595,3595,T,T,25,29,T,F,CharsOnly,,,Secret,1.50,4,F,F,F,Password:URL Credentials +35747,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2590,2590,T,F,25,29,T,F,CharsOnly,,,Secret,2.00,4,F,F,F,URL Credentials +35748,88763956,GitHub,6c73b80a,data/6c73b80a/test/88763956.go,3595,3595,T,T,25,29,T,F,CharsOnly,,,Secret,1.50,4,F,F,F,URL Credentials 35752,728925ac,GitHub,6c73b80a,data/6c73b80a/test/728925ac.go,2913,2913,F,F,16,24,F,F,,,,,0,0,F,F,F,Secret 35770,94ae447a,GitHub,6c73b80a,data/6c73b80a/src/94ae447a.go,696,696,F,F,,,F,F,,,,,0,0,F,F,F,Password:URL Credentials 35884,edc1c568,GitHub,6c73b80a,data/6c73b80a/test/edc1c568.go,540,540,F,F,8,64,F,F,Any,,,Unknown,4.63,56,F,F,F,Key @@ -800,4 +800,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1338398,9f7a2e57,GitHub,6c73b80a,data/6c73b80a/test/9f7a2e57.conf,24,24,T,F,59,516,F,F,,,,,0.0,0,F,F,F,JSON Web Token 1338475,728925ac,GitHub,6c73b80a,data/6c73b80a/test/728925ac.go,2964,2964,T,F,58,664,F,F,,,,,0.0,0,F,F,F,JSON Web Token 1338476,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,43,43,T,F,8,822,F,F,,,,,0.0,0,F,F,F,JSON Web Token - +1338569,515f4ca6,GitHub,6c73b80a,data/6c73b80a/test/515f4ca6.go,2580,2580,T,F,50,55,F,F,,,,,0.0,0,F,F,F,URL Credentials diff --git a/meta/8bc560cd.csv b/meta/8bc560cd.csv index 664852f49..dd947c64c 100644 --- a/meta/8bc560cd.csv +++ b/meta/8bc560cd.csv @@ -25,3 +25,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 107270,451af7c5,GitHub,8bc560cd,data/8bc560cd/src/451af7c5.rb,55,55,T,T,65,81,F,F,Any,,,Secret,4.21,27,F,F,F,Jira 2FA:Secret 107271,451af7c5,GitHub,8bc560cd,data/8bc560cd/src/451af7c5.rb,73,73,T,T,85,101,F,F,Any,,,Secret,4.41,41,F,F,T,Jira 2FA:Secret 107272,451af7c5,GitHub,8bc560cd,data/8bc560cd/src/451af7c5.rb,85,85,T,T,80,96,F,F,Any,,,Secret,4.29,39,F,F,F,Jira 2FA:Secret +1338572,451af7c5,GitHub,8bc560cd,data/8bc560cd/src/451af7c5.rb,67,67,F,F,31,108,F,F,,,,,0.0,0,F,F,F,Auth diff --git a/meta/ac9be8d9.csv b/meta/ac9be8d9.csv index dc8d47985..fe0b4fa3a 100644 --- a/meta/ac9be8d9.csv +++ b/meta/ac9be8d9.csv @@ -338,3 +338,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 134322,640c024a,GitHub,ac9be8d9,data/ac9be8d9/other/640c024a.md,307,307,F,F,,,F,F,,,,,0,0,F,F,F,Secret:Key 134323,f79c2c33,GitHub,ac9be8d9,data/ac9be8d9/src/f79c2c33.ex,73,73,F,F,,,F,F,,,,,0,0,F,F,F,Secret:Key 134324,f79c2c33,GitHub,ac9be8d9,data/ac9be8d9/src/f79c2c33.ex,74,74,F,F,,,F,F,,,,,0,0,F,F,F,Certificate +1338578,b42689a1,GitHub,ac9be8d9,data/ac9be8d9/test/b42689a1.exs,445,445,F,F,68,78,F,F,,,,,0.0,0,F,F,F,Password diff --git a/meta/ec138349.csv b/meta/ec138349.csv index 7b2c2b7e9..ef9e36104 100644 --- a/meta/ec138349.csv +++ b/meta/ec138349.csv @@ -182,4 +182,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value 1338528,18b43943,GitHub,ec138349,data/ec138349/test/18b43943.java,41,41,F,F,3,98,F,F,,,,,0.0,0,F,F,F,JSON Web Token 1338529,18b43943,GitHub,ec138349,data/ec138349/test/18b43943.java,44,44,F,F,3,98,F,F,,,,,0.0,0,F,F,F,JSON Web Token 1338530,18b43943,GitHub,ec138349,data/ec138349/test/18b43943.java,47,47,F,F,3,98,F,F,,,,,0.0,0,F,F,F,JSON Web Token - +1338573,2f9b15a9,GitHub,ec138349,data/ec138349/test/2f9b15a9.java,125,125,F,F,56,63,F,F,,,,,0.0,0,F,F,F,Auth:Token +1338575,2f9b15a9,GitHub,ec138349,data/ec138349/test/2f9b15a9.java,158,158,F,F,56,66,F,F,,,,,0.0,0,F,F,F,Auth:Token