Ubuntu update

README.md

@@ -38,7 +38,7 @@ If using this work, please cite [our paper](https://dl.acm.org/doi/10.1145/36347
 ### Run with Docker
 
 Running TLS traffic analyzer in docker is supported on the following host OS:
-- ubuntu:20.04
+- ubuntu:24.04
 - debian:11 (to come soon)
 - archlinux:latest (to come soon)
 You can use any of the name above in BASE_IMAGE build argument.
@@ -47,7 +47,7 @@ You can use any of the name above in BASE_IMAGE build argument.
 # Build the image with the same base OS as your host OS
 # WARNING: you must recompile the image on host kernel updates
 # Set BASE_IMAGE=... to one of the supported host OS (see above)
-docker build -t tls-traffic-analyzer:latest --no-cache --build-arg BASE_IMAGE=ubuntu:20.04 -f docker/Dockerfile .
+docker build -t tls-traffic-analyzer:latest --no-cache --build-arg BASE_IMAGE=ubuntu:24.04 -f docker/Dockerfile .
 
 # Get interface of default route (or set the interface you want to listen on)
 INTERFACE=$(ip -4 route | awk '/default/{print $5}')

docker/Dockerfile

@@ -1,20 +1,20 @@
-ARG BASE_IMAGE=ubuntu:20.04
+ARG BASE_IMAGE=ubuntu:24.04
 
 FROM $BASE_IMAGE
 
 # ARG must be after FROM
-ARG BCC_VERSION=v0.26.0
-ARG WIRESHARK_COMMIT=234f45d8e5e25918a2f517a79c5819949aa40da9
+ARG BCC_VERSION=v0.31.0
+ARG WIRESHARK_COMMIT=3b3b599037161e090a6d5747f685b211c54b4430
 
 # Multi host OS support
 RUN . /etc/os-release; \
-    if [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ]; then \
+    if [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "24.04" ]; then \
         apt-get update && \
         DEBIAN_FRONTEND=noninteractive apt-get install -y \
         build-essential cmake linux-headers-$(uname -r) \
-        bison build-essential cmake flex git libedit-dev \
-        libllvm12 llvm-12-dev libclang-12-dev python zlib1g-dev libelf-dev libfl-dev python3-distutils \
-        python3-pip \
+        bison cmake flex git libedit-dev \
+        libllvm18 llvm-18-dev libclang-18-dev libpolly-18-dev python3 zlib1g-dev libelf-dev libfl-dev python3-setuptools \
+        python3-pip libspeexdsp-dev zip \
         curl libglib2.0-dev libgcrypt-dev libc-ares-dev gnutls-dev libpcap-dev && \
         rm -rf /var/lib/apt/lists/*; \
     elif [ "$ID" = "debian" ] && [ "$VERSION_ID" = "11" ]; then \ 
@@ -71,16 +71,28 @@ RUN \
     make install && \
     cd /tmp && \
     rm -rf /tmp/bcc
+#RUN \
+#    cd /tmp && \
+#    git clone https://github.com/iovisor/bcc.git \
+#    mkdir bcc/build; cd bcc/build \
+#    cmake .. \
+#    make \
+#    sudo make install \
+#    cmake -DPYTHON_CMD=python3 .. # build python3 binding \
+#    pushd src/python/ \
+#    make \
+#    sudo make install \
+#    popd
 
 # Install Python dependencies
 COPY requirements.txt /opt
-RUN pip3 install -r /opt/requirements.txt
+RUN pip3 install -r /opt/requirements.txt --break-system-packages
 
 # Add /opt/wireshark-custom/bin/ to PATH
-ENV PATH /opt/wireshark-custom/bin:$PATH
+ENV PATH=/opt/wireshark-custom/bin:$PATH
 
 # Copy and compile (Cython) code
-ENV PYTHONPATH /app
+ENV PYTHONPATH=/app
 WORKDIR /app
 
 COPY setup.py /app

patchs/tshark.patch

@@ -1,25 +1,26 @@
 diff --git a/epan/dissectors/packet-tls.c b/epan/dissectors/packet-tls.c
-index 737a913..d00fdb5 100644
+index 9bf935021d..2f3736fa0a 100644
 --- a/epan/dissectors/packet-tls.c
 +++ b/epan/dissectors/packet-tls.c
-@@ -84,6 +84,16 @@ static gboolean tls_desegment          = TRUE;
- static gboolean tls_desegment_app_data = TRUE;
- static gboolean tls_ignore_mac_failed  = FALSE;
+@@ -87,6 +87,17 @@ static bool tls_desegment          = true;
+ static bool tls_desegment_app_data = true;
+ static bool tls_ignore_mac_failed;
 
 +// Variables to store secrets for TLS 1.3 as decryption is done in 2 steps (for each direction)
 +#define TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES 48
-+static gint tls13_client_secret_position = -1;  // -1 means not found
-+static gint tls13_server_secret_position = -1;
-+static guchar tls13_app_traffic_client_secret[TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES];
-+static guchar tls13_app_traffic_server_secret[TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES];
++static int tls13_client_secret_position = -1;  // -1 means not found
++static int tls13_server_secret_position = -1;
++static unsigned char tls13_app_traffic_client_secret[TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES];
++static unsigned char tls13_app_traffic_server_secret[TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES];
 +
 +// Load app traffic keys on first app data record for each direction
-+static gboolean first_app_data_record_from_client_seen = FALSE;
-+static gboolean first_app_data_record_from_server_seen = FALSE;
-
- /*********************************************************************
-  *
-@@ -1920,6 +1930,175 @@ dissect_ssl3_record(tvbuff_t *tvb, packet_info *pinfo,
++static bool first_app_data_record_from_client_seen = false;
++static bool first_app_data_record_from_server_seen = false;
++
+ #define PORT_HEUR_DEFAULT "443"
+ /* Try heuristic dissectors before dissectors assigned to a port.
+  * Dissectors assigned via ALPN always take precedence. */
+@@ -2194,6 +2205,176 @@ dissect_ssl3_record(tvbuff_t *tvb, packet_info *pinfo,
       */
      ssl_debug_printf("dissect_ssl3_record: content_type %d %s\n",content_type, val_to_str_const(content_type, ssl_31_content_type, "unknown"));
 
@@ -31,16 +32,16 @@ index 737a913..d00fdb5 100644
 +
 +        FILE   *fp;
 +        size_t read;
-+        gboolean decryption_success = FALSE;
-+        guchar master_secret_candidate[SSL_MASTER_SECRET_LENGTH];
++        bool decryption_success = false;
++        unsigned char master_secret_candidate[SSL_MASTER_SECRET_LENGTH];
 +        int key_position = 0;
 +
 +        // BRUTEFORCE_TLS12_FILE is expected to be a binary file with all key candidates concatenated
 +        fp = fopen(getenv("BRUTEFORCE_TLS12_FILE"), "rb");
 +        if (fp == NULL) {
 +            perror("Fail to open key candidates file");
 +            exit(EXIT_FAILURE);
-+        }       
++        }
 +
 +        while (!feof(fp)) {
 +
@@ -102,18 +103,18 @@ index 737a913..d00fdb5 100644
 +
 +        FILE   *fp;
 +        size_t read;
-+        gboolean decryption_success = FALSE;
++        bool decryption_success = false;
 +        StringInfo secret;
-+        guchar key_candidate[TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES];
-+        gint key_position = 0;
++        unsigned char key_candidate[TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES];
++        int key_position = 0;
 +
 +        // BRUTEFORCE_TLS13_FILE is expected to be a binary file with all key candidates concatenated
 +        fp = fopen(getenv("BRUTEFORCE_TLS13_FILE"), "rb");
 +        if (fp == NULL) {
 +            perror("Fail to open key candidates file");
 +            exit(EXIT_FAILURE);
 +        }
-+        
++
 +        ssl_data_alloc(&secret, TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES);
 +
 +        while (!feof(fp)) {
@@ -125,7 +126,7 @@ index 737a913..d00fdb5 100644
 +            ssl_data_set(&secret, key_candidate, TLS13_APP_TRAFFIC_SECRET_LENGTH_BYTES);
 +
 +            // Re-generate encryption keys, IVs, etc. with PRF
-+            if (tls13_generate_keys(ssl, &secret, is_from_server) < 0) {
++            if (!tls13_generate_keys(ssl, &secret, is_from_server)) {
 +                ssl_debug_printf("Fail to generate keyring material\n");
 +                exit(EXIT_FAILURE);
 +            }
@@ -135,7 +136,7 @@ index 737a913..d00fdb5 100644
 +                    content_type, record_version, record_length,
 +                    content_type == SSL_ID_APP_DATA ||
 +                    content_type == SSL_ID_HANDSHAKE, curr_layer_num_ssl);
-+            
++
 +            if (decryption_success) {
 +                if (is_from_server) {
 +                    printf("INFO Server key found\n");
@@ -185,12 +186,13 @@ index 737a913..d00fdb5 100644
 +    */
 +    if (content_type == SSL_ID_APP_DATA && !first_app_data_record_from_client_seen && !is_from_server && ssl) {
 +        tls13_change_key(ssl, &ssl_master_key_map, is_from_server, TLS_SECRET_APP);
-+        first_app_data_record_from_client_seen = TRUE;
++        first_app_data_record_from_client_seen = true;
 +    }
 +    if (content_type == SSL_ID_APP_DATA && !first_app_data_record_from_server_seen && is_from_server && ssl) {
 +        tls13_change_key(ssl, &ssl_master_key_map, is_from_server, TLS_SECRET_APP);
-+        first_app_data_record_from_server_seen = TRUE;
++        first_app_data_record_from_server_seen = true;
 +    }
++
 +
      /* try to decrypt record on the first pass, if possible. Store decrypted
       * record for later usage (without having to decrypt again). The offset is

requirements.txt

@@ -1,7 +1,7 @@
-pyshark==0.5.3
-ujson==5.4.0
-scapy==2.4.5
-cryptography==37.0.4
-tqdm==4.64.0
-docker==6.0.0
-cython==0.29.32
+pyshark==0.6
+ujson==5.10.0
+scapy==2.6.0
+cryptography==43.0.3
+tqdm==4.67.0
+docker==7.1.0
+cython==3.0.11

src/baseline/entropy_filter.py

@@ -4,7 +4,6 @@
 
 import logging
 import math
-from typing import Dict
 
 from tqdm import tqdm
 
@@ -25,7 +24,7 @@ def entropy(string: str) -> float:
     return -sum([p * math.log(p) / math.log(2.0) for p in prob])
 
 
-def baseline_entropy_filter(snapshot: MemorySnapshot, entropy_threshold: float = 3.6) -> Dict[str, any]:
+def baseline_entropy_filter(snapshot: MemorySnapshot, entropy_threshold: float = 3.6) -> dict[str, any]:
     """
     Return figures about the baseline approach
     """

src/dumper/handshake_detector.c

@@ -66,10 +66,12 @@ int trace_ip4_datagram_connect(struct pt_regs *ctx, struct sock *skp, struct soc
     return 0;
   }
 
-  struct inet_sock *inet = inet_sk(skp);
+  //struct inet_sock *inet = inet_sk(skp); // This somehow makes the compilation fail despite the same macro working in read_ipv4_tuple when replacing the dodgy cast with inet_sk()...
+  //u32 saddr = bpf_ntohl(inet->inet_saddr);  // FIXME: equal to 0?
+  //u16 sport = bpf_ntohs(inet->inet_sport);
   u32 daddr = bpf_ntohl(sa_in->sin_addr.s_addr);
-  u32 saddr = bpf_ntohl(inet->inet_saddr);  // FIXME: equal to 0?
-  u16 sport = bpf_ntohs(inet->inet_sport);
+  u32 saddr = bpf_ntohl(skp->__sk_common.skc_rcv_saddr);  // FIXME: equal to 0?
+  u16 sport = skp->sk_num;
 
   struct ipv4_tuple_t tuple = {};
   tuple.saddr = saddr;

src/dumper/handshake_detector.py

@@ -11,7 +11,6 @@
 import time
 from collections import deque
 from ipaddress import ip_address
-from typing import Dict, Tuple, Union
 
 from src.baseline.entropy_filter import baseline_entropy_filter
 from src.dumper.bpf import QuicEvent, TlsEvent, setup_bpf
@@ -28,9 +27,9 @@ class MemoryDumper():
     def __init__(
             self,
             interface: str,
-            tls_sessions: Dict[str, TLSSession],
+            tls_sessions: dict[str, TLSSession],
             max_workers: int = 5,
-            allowed_commands: Tuple[str] = None) -> None:
+            allowed_commands: tuple[str] | None = None) -> None:
         """
         interface: (str) Interface to listen on
         tls_sessions: (dict) fork-safe dict to store TLS sessions
@@ -52,18 +51,18 @@ def __init__(
         self.allowed_commands = allowed_commands if allowed_commands != "*" else None
 
         # key is pid, value is a MemoryDiffer
-        self.memdiffers: Dict[int, MemoryDiffer] = self.manager.dict()
+        self.memdiffers: dict[int, MemoryDiffer] = self.manager.dict()
 
         # Keep trace of stopped processes
         # key is pid, value is a counter representing the number of memory dumps ongoing
-        self.stopped_pid: Dict[int, int] = self.manager.dict()
+        self.stopped_pid: dict[int, int] = self.manager.dict()
 
         # Keep trace when process was stopped
         # key is pid, value is a counter returned by time.perf_counter_ns()
-        self.stop_ts: Dict[int, int] = self.manager.dict()
+        self.stop_ts: dict[int, int] = self.manager.dict()
 
         # Per PID lock for memory operations
-        self.locks: Dict[int, self.manager.Lock] = self.manager.dict()
+        self.locks: dict[int, self.manager.Lock] = self.manager.dict()
 
         # Prevent duplicate events
         self.events = deque(maxlen=100)
@@ -106,7 +105,7 @@ def close(self):
                 os.kill(pid, signal.SIGCONT)
 
 
-    def process_tls_event(self, event_type: str, event: Union[TlsEvent, QuicEvent]) -> None:
+    def process_tls_event(self, event_type: str, event: TlsEvent|QuicEvent) -> None:
         """
         Process BPF event
         """

src/keyfinder/finder.py

@@ -12,9 +12,9 @@
 import shutil
 import subprocess
 import time
-from tempfile import NamedTemporaryFile
-from typing import Dict, List, Tuple, Union
+from datetime import datetime
 
+from multiprocessing.managers import DictProxy
 from src.keyfinder.tshark_keytester import TsharkKeyTester
 from src.models import TLSSession
 from tqdm import tqdm
@@ -29,12 +29,12 @@ class KeyFinder():
     """
     Brute force TLS keys from memory dump
     """
-    def __init__(self, tls_sessions: Dict[str, TLSSession], dump_directory: str) -> None:
+    def __init__(self, tls_sessions: DictProxy[str, TLSSession], dump_directory: str) -> None:
         self.tls_sessions = tls_sessions
         self.dump_directory = dump_directory
 
         # Cache for SHA256 hashes of commands (e.g. SHA256 of /usr/bin/curl)
-        self.command_hashes: Dict[str, str] = {}
+        self.command_hashes: dict[str, str] = {}
 
     def close(self) -> None:
         pass
@@ -69,7 +69,7 @@ def run(self) -> None:
 
             time.sleep(1)
 
-    def find_key(self, tls_session: TLSSession) -> Union[str, None]:
+    def find_key(self, tls_session: TLSSession) -> str | None:
         """
         Return keys in SSLKEYLOGFILE format (string) or None if not found
         """
@@ -153,7 +153,7 @@ def find_key(self, tls_session: TLSSession) -> Union[str, None]:
             keytester.close()
 
 
-    def get_key_candidates(self, diff_hex: str, cmd: str, entropy_threshold: float = 3.5) -> List[str]:
+    def get_key_candidates(self, diff_hex: str, cmd: str, entropy_threshold: float = 3.5) -> list[str]:
         """
         Return list of key candidates ordered from most likely to less likely
         """
@@ -172,7 +172,7 @@ def store_stats(
             self,
             diff_hex: str,
             tls_session: TLSSession,
-            keytester_results: Dict[str, Union[bool, str, int]],
+            keytester_results: dict[str, bool|str|int],
             key_candidates_length: int) -> None:
         """
         Store stats
@@ -263,7 +263,7 @@ def store_stats(
             fd.write(json.dumps(stats) + "\n")
 
     @staticmethod
-    def get_context(diff_hex: str, key: str, context_size: int = 100) -> Tuple[int, str, str]:
+    def get_context(diff_hex: str, key: str, context_size: int = 100) -> tuple[int, str, str]:
         """
         Get pre/post key content
         """
@@ -285,7 +285,7 @@ def get_context(diff_hex: str, key: str, context_size: int = 100) -> Tuple[int,
         return start_key_index, pre_key, post_key
 
     @staticmethod
-    def get_memory_region_path(key_hex: str, tls_session: TLSSession) -> Union[str, None]:
+    def get_memory_region_path(key_hex: str, tls_session: TLSSession) -> str | None:
         """
         Return the path of the region where the key was stored or None if not found
         """
@@ -302,7 +302,7 @@ def entropy(string: str) -> float:
         prob = [float(string.count(c)) / len(string) for c in dict.fromkeys(list(string))]
         return -sum([p * math.log(p) / math.log(2.0) for p in prob])
 
-    def get_command_sha256(self, command: str) -> Union[str, None]:
+    def get_command_sha256(self, command: str) -> str | None:
         """
         Return the SHA256 hash of the binary of command
         """
@@ -323,8 +323,8 @@ def get_command_sha256(self, command: str) -> Union[str, None]:
 
         return binary_hash
 
-    @staticmethod
     def edit_pcap(
+        self,
         dump_file: str,
         ssl_key_log_file_content: str,
         capture_comment: str) -> None:
@@ -335,8 +335,9 @@ def edit_pcap(
             os.environ.get("CUSTOM_WIRESHARK_BIN_PATH", "/opt/wireshark-custom/bin"),
             "editcap"
         )
-        with NamedTemporaryFile(buffering=0) as keylog: 
-            keylog.write(ssl_key_log_file_content.encode("ascii"))
+        with open(os.path.join(self.dump_directory, os.environ.get("SSLKEYLOG_FILENAME", f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}_sslkeylogfile")), "wt") as keylog:
+            _ = keylog.write(ssl_key_log_file_content)
+            keylog.close()
             try:
                 subprocess.run(
                     [