feat: add better functionality to secret detection

add the following functionality:
- improve regexes for identificating secrets via patterns
- add high entropy secret detection
- include the key of the secret in the found matches

Closes: #209

Author: yrobla

prompts/default.yaml

@@ -13,11 +13,13 @@ default_chat: |
   - Pkg 2: [https://www.insight.stacklok.com/report/ecosystem/package_name](https://www.insight.stacklok.com/report/ecosystem/package_name?utm_source=codegate)
   - ...
 
-  If you are Kodu, also append that when finding malicious packages or leaked secrets:
+  Also append that when finding malicious packages or leaked secrets:
 
-  "** critical vulnerability found, you must take action **
-  Please notify the user with ask follow up tool about the following vulnerability:
-  <vulnerability_content>"
+  "** critical vulnerability found, you must take action **"
+
+  If you are Kodu, also append:
+  
+  "Please notify the user with ask follow up tool about the following vulnerability: <vulnerability_content>"
 
   If package_name contains any special chars, please encode them using URL encoding.
 

signatures.yaml

@@ -32,6 +32,10 @@
     - API Key: \bSK[0-9a-fA-F]{32}\b
     - Access Token: \b(?i)55[0-9a-fA-F]{32}\b
 
+- Twitter:
+    - Client ID: (?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}
+    - Secret: (?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}
+
 - Atlassian:
     - JIRA Token: (?i)(jira_token)
     - Bitbucket Data Center Access Token: BBDC-[a-zA-Z0-9+]{44}
@@ -40,6 +44,7 @@
 
 - Google:
     - Cloud API Key: AIza[0-9A-Za-z_-]{35}
+    - Cloud Platform API Key: (?i)(google|gcp|youtube|drive|yt)(.{0,20})?['\"][AIza[0-9a-z\\-_]{35}]['\"]
     - Cloud OAuth Secret: (?i)(GOCSPX-[-0-9A-Za-z_]{24,32})
     #- reCaptcha Key: 6L([A-Za-z0-9_-]{6})AAAAA([A-Za-z0-9_-]{27})
     - OAuth Key: ya29\.[0-9A-Za-z_-]{64,256}
@@ -98,6 +103,8 @@
 - Meta:
     - Page Access Token: (?i)(EAAG[0-9A-Za-z]{10,128})
     - Facebook Access Token: EAACEdEose0cBA[0-9A-Za-z]+
+    - Facebook Client ID: (?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}
+    - Facebook Secret Key: (?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}
     #- Client Token: (?i)fb[a-zA-Z0-9]{24,32}
     - Instagram Access Token: (?i)(IGQV[0-9A-Za-z-_]{10,255})
     - Instagram App Secret: (?i)(ig_[a-f0-9]{32})
@@ -192,6 +199,7 @@
 
 - Artifactory:
     - Token: AKCp[0-9][a-zA-Z0-9]{64,128}
+    - Password: AP[\dABCDEF][a-zA-Z0-9]{8,}
 
 - Figma:
     - Personal Access Token: (figd_[a-zA-Z0-9-_]{14,32}_[a-zA-Z0-9-_]{14,32})
@@ -265,13 +273,6 @@
 - Postgresql:
     - URL: (?i)(?:pgsql:|postgres:|postgresql:)//[\S]{1,256}:[\S]{1,256}@[-.%\w\/:]+\.[\S]+
 
-- GitHub:
-    - Access Token: (?i)\bghp_[A-Za-z0-9]{36}\b
-    - OAuth Token: (?i)\bgho_[A-Za-z0-9]{36}\b
-    - App Installation Token: (?i)\bghu_[A-Za-z0-9]{36}\b
-    - App user Token: (?i)\bghs_[A-Za-z0-9]{36}\b
-    - Refresh Token: (?i)\bghr_[A-Za-z0-9]{36}\b
-
 - Addresses:
     - Bitcoin Legacy: \b[13][a-km-zA-HJ-NP-Z1-9]{25,34}\b
     - Bitcoin SegWit: \b(bc1)[a-zA-HJ-NP-Z0-9]{39,59}\b
@@ -299,7 +300,8 @@
     - Advanced Message Queuing Protocol (AMQP) URL: amqp://[a-zA-Z0-9-_+.@]+:[^@]+@[^/]+
     # Private Keys
     - JSON Web Key Block: /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/gm
-    - Private Key Block: -{0,5} ?BEGIN (?:RSA |ENCRYPTED |OPENSSH |SSH2 )?PRIVATE KEY ?-{0,5} ?([\s\S]*?)-{0,5} ?END (?:RSA |ENCRYPTED |OPENSSH |SSH2 )?PRIVATE KEY ?-{0,5}
+    - Private Key Block: -{0,5} ?BEGIN (?:RSA |ENCRYPTED |OPENSSH |SSH2 |DSA |EC )?PRIVATE KEY ?-{0,5} ?([\s\S]*?)-{0,5} ?END (?:RSA |ENCRYPTED |OPENSSH |SSH2 |DSA |EC )?PRIVATE KEY ?-{0,5}
+    - PGP: -{0,5}BEGIN PGP PRIVATE KEY BLOCK-{0,5}[\s\S]*?-{0,5}END PGP PRIVATE KEY BLOCK-{0,5}
     - Bitcoin Private Key: \b[5KL][1-9A-HJ-NP-Za-km-z]{50,51}\b
     - Ethereum Private Key: \b0x[a-fA-F0-9]{64}\b
     - Litecoin Private Key: \b[5KL][1-9A-HJ-NP-Za-km-z]{50,51}\b

src/codegate/pipeline/secrets/secrets.py

@@ -147,6 +147,7 @@ def obfuscate(self, text: str) -> tuple[str, List[Match]]:
             logger.info(
                 f"\nService: {match.service}"
                 f"\nType: {match.type}"
+                f"\nKey: {match.key}"
                 f"\nOriginal: {match.value}"
                 f"\nEncrypted: {hidden_secret}"
             )
@@ -450,6 +451,7 @@ async def process_chunk(
             or input_context.metadata.get("redacted_secrets_count", 0) == 0
         ):
             return [chunk]
+
         tool_name = next(
             (
                 tool.lower()

src/codegate/pipeline/secrets/signatures.py

@@ -1,4 +1,5 @@
 # signatures.py
+import math
 import re
 from pathlib import Path
 from threading import Lock
@@ -15,6 +16,7 @@ class Match(NamedTuple):
 
     service: str
     type: str
+    key: str
     value: str
     line_number: int
     start_index: int
@@ -42,6 +44,16 @@ class CodegateSignatures:
     _signature_groups: ClassVar[List[SignatureGroup]] = []
     _compiled_regexes: ClassVar[Dict[str, re.Pattern]] = {}
     _yaml_path: ClassVar[Optional[str]] = None
+    HIGH_ENTROPY_THRESHOLD: ClassVar[float] = 4.0
+
+    @classmethod
+    def _calculate_entropy(cls, text: str) -> float:
+        """Calculate Shannon entropy for a given string."""
+        if not text:
+            return 0.0
+
+        prob = {char: text.count(char) / len(text) for char in set(text)}
+        return -sum(p * math.log2(p) for p in prob.values())
 
     @classmethod
     def reset(cls) -> None:
@@ -180,22 +192,11 @@ def _load_signatures(cls) -> None:
             # Clear existing signatures before loading new ones
             cls._signature_groups = []
             cls._compiled_regexes = {}
-
             yaml_data = cls._load_yaml(cls._yaml_path)
 
-            # Add custom GitHub token patterns
-            github_patterns = {
-                "Access Token": r"ghp_[0-9a-zA-Z]{32}",
-                "Personal Token": r"github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}",
-            }
-            cls._add_signature_group("GitHub", github_patterns)
-
             # Process patterns from YAML
             for item in yaml_data:
                 for service_name, patterns in item.items():
-                    if service_name == "GitHub":
-                        continue
-
                     service_patterns = {}
                     for pattern_dict in patterns:
                         for pattern_name, pattern in pattern_dict.items():
@@ -224,6 +225,7 @@ def find_in_string(cls, text: Union[str, List[str]]) -> List[Match]:
             raise RuntimeError("SecretFinder not initialized.")
 
         matches = []
+        found_values = set()
 
         # Split text into lines for processing
         try:
@@ -233,32 +235,74 @@ def find_in_string(cls, text: Union[str, List[str]]) -> List[Match]:
             return []
 
         for line_num, line in enumerate(lines, start=1):
-            for group in cls._signature_groups:
-                for pattern_name in group.patterns:
-                    regex_key = f"{group.name}:{pattern_name}"
-                    regex = cls._compiled_regexes.get(regex_key)
+            matches.extend(cls._find_regex_matches(line, line_num, found_values))
+            matches.extend(cls._find_high_entropy_matches(line, line_num, found_values))
+        return matches
 
-                    if not regex:
+    @classmethod
+    def _find_regex_matches(cls, line: str, line_num: int, found_values: set) -> List[Match]:
+        """Find matches using regex patterns."""
+        matches = []
+        for group in cls._signature_groups:
+            for pattern_name, regex in group.patterns.items():
+                regex_key = f"{group.name}:{pattern_name}"
+                regex = cls._compiled_regexes.get(regex_key)
+                if not regex:
+                    continue
+                for match in regex.finditer(line):
+                    value = match.group()
+                    key = cls._extract_key_from_line(line, value)
+                    pattern = f"{key}:{value}"
+                    if value.lower() == "token" or pattern in found_values:
                         continue
+                    found_values.add(pattern)
+                    matches.append(
+                        Match(
+                            group.name,
+                            pattern_name,
+                            key,
+                            value,
+                            line_num,
+                            match.start(),
+                            match.end(),
+                        )
+                    )
+        return matches
 
-                    try:
-                        for match in regex.finditer(line):
-                            value = match.group()
-                            if value.lower() == "token":
-                                continue
+    @staticmethod
+    def _extract_key_from_line(line: str, secret_value: str) -> Optional[str]:
+        """
+        Extract the key associated with a secret value if it follows a key=value pattern.
+        """
+        match = re.search(
+            r'([A-Za-z_][A-Za-z0-9_]*)\s*=\s*["\']?' + re.escape(secret_value) + r'["\']?', line
+        )
+        return match.group(1) if match else None
 
-                            matches.append(
-                                Match(
-                                    service=group.name,
-                                    type=pattern_name,
-                                    value=value,
-                                    line_number=line_num,
-                                    start_index=match.start(),
-                                    end_index=match.end(),
-                                )
-                            )
-                    except Exception as e:
-                        logger.warning(f"Error matching pattern {regex_key}: {e}")
-                        continue
+    @classmethod
+    def _find_high_entropy_matches(cls, line: str, line_num: int, found_values: set) -> List[Match]:
+        """Find matches based on high entropy values."""
+        matches = []
+        assignment_pattern = re.findall(
+            r"([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([\"']?([A-Za-z0-9_\-\.+/=]{8,})[\"']?)", line
+        )
+
+        for key, _, word in assignment_pattern:
+            pattern = f"{key}:{word}"
+            if pattern in found_values or word.startswith("REDACTED"):
+                continue
+            if cls._calculate_entropy(word) >= cls.HIGH_ENTROPY_THRESHOLD:
+                found_values.add(pattern)
+                matches.append(
+                    Match(
+                        "High Entropy",
+                        "Potential Secret",
+                        key,
+                        word,
+                        line_num,
+                        line.find(word),
+                        line.find(word) + len(word),
+                    )
+                )
 
         return matches

tests/pipeline/secrets/test_secrets.py

@@ -78,6 +78,7 @@ def test_hide_secret(self):
         match = Match(
             service="AWS",
             type="Access Key",
+            key="API_KEY",
             value="AKIAIOSFODNN7EXAMPLE",
             line_number=1,
             start_index=0,
@@ -115,6 +116,7 @@ def test_hide_secret(self):
         match = Match(
             service="AWS",
             type="Access Key",
+            key="API_KEY",
             value="AKIAIOSFODNN7EXAMPLE",
             line_number=1,
             start_index=0,
@@ -129,6 +131,8 @@ def test_obfuscate(self):
         # Test text with multiple secrets
         text = "API_KEY=AKIAIOSFODNN7EXAMPLE\nPASSWORD=AKIAIOSFODNN7EXAMPLE"
         protected, matched_secrets = self.obfuscator.obfuscate(text)
+        print(protected)
+        print(matched_secrets)
 
         assert len(matched_secrets) == 2
         assert "AKIAIOSFODNN7EXAMPLE" not in protected