Merge branch 'dev' into config2

Author: IvanNardi

.github/workflows/build.yml

@@ -89,15 +89,15 @@ jobs:
         nBPF: [""]
         lto_gold_linker: [""]
         include:
-          - compiler: "gcc-7" # "Oldest" gcc easily available
+          - compiler: "gcc-4.9" # "Oldest" gcc easily available. To simulate RHEL7
             os: ubuntu-20.04
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
-          - compiler: "gcc-12" # "Newest" gcc easily available
+          - compiler: "gcc-13" # "Newest" gcc easily available
             os: ubuntu-22.04
             arch: "x86_64"
             gcrypt: ""
@@ -114,9 +114,9 @@ jobs:
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
-          - compiler: "clang-14" # "Newest" clang easily available
-            ar: "llvm-ar-14"
-            ranlib: "llvm-ranlib-14"
+          - compiler: "clang-17" # "Newest" clang easily available
+            ar: "llvm-ar-17"
+            ranlib: "llvm-ranlib-17"
             os: ubuntu-22.04
             arch: "x86_64"
             gcrypt: ""
@@ -141,7 +141,7 @@ jobs:
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: "nBPF"
-          - compiler: "clang-14"
+          - compiler: "clang-17"
             os: ubuntu-22.04
             arch: "x86_64"
             gcrypt: ""
@@ -226,9 +226,22 @@ jobs:
           make
           cd -
       - name: Setup Ubuntu specified compiler
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && ! startsWith(matrix.compiler, 'cc')
-        run: |
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && ! startsWith(matrix.compiler, 'cc') && ! startsWith(matrix.compiler, 'clang-17')
+        run: |
+          #For gcc-4.9 (on ubuntu-20.04)
+          echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" | sudo tee -a /etc/apt/sources.list
+          echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" | sudo tee -a /etc/apt/sources.list
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
+          #For gcc-13 (on ubuntu-22.04)
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/ppa
+          sudo apt-get update
           sudo apt-get install ${{ matrix.compiler }}
+      - name: Setup Ubuntu specified (newest) compiler
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'clang-17')
+        run: |
+          wget https://apt.llvm.org/llvm.sh
+          chmod u+x llvm.sh
+          sudo ./llvm.sh 17
       - name: Install Windows msys2 prerequisites
         if: startsWith(matrix.os, 'windows')
         uses: msys2/setup-msys2@v2

configure.ac

@@ -64,8 +64,14 @@ AS_IF([test "${with_sanitizer+set}" = set -o "${with_thread_sanitizer+set}" = se
 ])
 
 AS_IF([test "${with_sanitizer+set}" = set],[
-  NDPI_CFLAGS="${NDPI_CFLAGS} -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=leak -fno-omit-frame-pointer"
-  NDPI_LDFLAGS="${NDPI_LDFLAGS} -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=leak"
+  NDPI_CFLAGS="${NDPI_CFLAGS} -fsanitize=address -fsanitize=undefined -fsanitize=leak -fno-omit-frame-pointer"
+  NDPI_LDFLAGS="${NDPI_LDFLAGS} -fsanitize=address -fsanitize=undefined -fsanitize=leak"
+  #Sanitizers should work on any compilers that we support (or that we test on CI, at least)
+  #Exception: "-fsanitize=alignment" is not supported in gcc 4.9
+  AX_CHECK_COMPILE_FLAG([-fno-sanitize=alignment], [
+    NDPI_CFLAGS="${NDPI_CFLAGS} -fno-sanitize=alignment"
+    NDPI_LDFLAGS="${NDPI_LDFLAGS} -fno-sanitize=alignment"
+  ])
 ])
 
 AS_IF([test "${with_thread_sanitizer+set}" = set],[

doc/protocols.rst

@@ -377,3 +377,34 @@ References: `Protocol Specs: <https://uftp-multicast.sourceforge.net/protocol.tx
 OpenFlow protocol is a network protocol closely associated with Software-Defined Networking (SDN).
 
 References: `Protocol Specs: <https://opennetworking.org/wp-content/uploads/2014/10/openflow-switch-v1.5.1.pdf>`_.
+
+
+.. _Proto 375:
+
+`NDPI_PROTOCOL_JSON_RPC`
+======================
+JSON-RPC is a remote procedure call protocol encoded in JSON.
+
+References: `Protocol Specs: <https://www.jsonrpc.org/specification>`_.
+
+
+.. _Proto 376:
+
+`NDPI_PROTOCOL_WEBDAV`
+======================
+WebDAV is a set of extensions to the HTTP protocol that allows WebDAV clients to collaboratively edit and manage files on remote Web servers.
+
+References: `RFC4918: <https://datatracker.ietf.org/doc/html/rfc4918>`_.
+
+Notes:
+
+- WebDAV is almost always encrypted, i.e. transported over TLS.
+
+
+.. _Proto 377:
+
+`NDPI_PROTOCOL_APACHE_KAFKA`
+======================
+Apache Kafka is a distributed event store and stream-processing platform.
+
+References: `Official site <https://kafka.apache.org>`_ and `Github <https://github.com/apache/kafka>`_.

example/ndpiSimpleIntegration.c

@@ -226,6 +226,7 @@ static void ndpi_flow_info_freer(void * const node)
 static void free_workflow(struct nDPI_workflow ** const workflow)
 {
   struct nDPI_workflow * const w = *workflow;
+  size_t i;
 
   if (w == NULL) {
     return;
@@ -239,7 +240,7 @@ static void free_workflow(struct nDPI_workflow ** const workflow)
   if (w->ndpi_struct != NULL) {
     ndpi_exit_detection_module(w->ndpi_struct);
   }
-  for(size_t i = 0; i < w->max_active_flows; i++) {
+  for(i = 0; i < w->max_active_flows; i++) {
     ndpi_tdestroy(w->ndpi_flows_active[i], ndpi_flow_info_freer);
   }
   ndpi_free(w->ndpi_flows_active);
@@ -268,6 +269,7 @@ static int setup_reader_threads(char const * const file_or_device)
 {
   char * file_or_default_device;
   char pcap_error_buffer[PCAP_ERRBUF_SIZE];
+  int i;
 
   if (reader_thread_count > MAX_READER_THREADS) {
     return 1;
@@ -286,7 +288,7 @@ static int setup_reader_threads(char const * const file_or_device)
     }
   }
 
-  for (int i = 0; i < reader_thread_count; ++i) {
+  for (i = 0; i < reader_thread_count; ++i) {
     reader_threads[i].workflow = init_workflow(file_or_default_device);
     if (reader_threads[i].workflow == NULL)
       {
@@ -496,8 +498,10 @@ static int ndpi_workflow_node_cmp(void const * const A, void const * const B) {
 
 static void check_for_idle_flows(struct nDPI_workflow * const workflow)
 {
+  size_t idle_scan_index;
+
   if (workflow->last_idle_scan_time + IDLE_SCAN_PERIOD < workflow->last_time) {
-    for (size_t idle_scan_index = 0; idle_scan_index < workflow->max_active_flows; ++idle_scan_index) {
+    for (idle_scan_index = 0; idle_scan_index < workflow->max_active_flows; ++idle_scan_index) {
       ndpi_twalk(workflow->ndpi_flows_active[idle_scan_index], ndpi_idle_scan_walker, workflow);
 
       while (workflow->cur_idle_flows > 0) {
@@ -526,7 +530,7 @@ static void ndpi_process_packet(uint8_t * const args,
   struct nDPI_reader_thread * const reader_thread =
     (struct nDPI_reader_thread *)args;
   struct nDPI_workflow * workflow;
-  struct nDPI_flow_info flow = {};
+  struct nDPI_flow_info flow;
 
   size_t hashed_index;
   void * tree_result;
@@ -547,6 +551,8 @@ static void ndpi_process_packet(uint8_t * const args,
   uint16_t type;
   uint32_t thread_index = INITIAL_THREAD_HASH; // generated with `dd if=/dev/random bs=1024 count=1 |& hd'
 
+  memset(&flow, '\0', sizeof(flow));
+
   if (reader_thread == NULL) {
     return;
   }
@@ -1024,7 +1030,9 @@ static void * processing_thread(void * const ndpi_thread_arg)
 
 static int processing_threads_error_or_eof(void)
 {
-  for (int i = 0; i < reader_thread_count; ++i) {
+  int i;
+
+  for (i = 0; i < reader_thread_count; ++i) {
     if (__sync_fetch_and_add(&reader_threads[i].workflow->error_or_eof, 0) == 0) {
       return 0;
     }
@@ -1034,6 +1042,8 @@ static int processing_threads_error_or_eof(void)
 
 static int start_reader_threads(void)
 {
+  int i;
+
 #ifndef WIN32
   sigset_t thread_signal_set, old_signal_set;
 
@@ -1046,7 +1056,7 @@ static int start_reader_threads(void)
   }
 #endif
 
-  for (int i = 0; i < reader_thread_count; ++i) {
+  for (i = 0; i < reader_thread_count; ++i) {
     reader_threads[i].array_index = i;
 
     if (reader_threads[i].workflow == NULL) {
@@ -1072,20 +1082,21 @@ static int start_reader_threads(void)
 
 static int stop_reader_threads(void)
 {
+  int i;
   unsigned long long int total_packets_captured = 0;
   unsigned long long int total_packets_processed = 0;
   unsigned long long int total_l4_data_len = 0;
   unsigned long long int total_flows_captured = 0;
   unsigned long long int total_flows_idle = 0;
   unsigned long long int total_flows_detected = 0;
 
-  for (int i = 0; i < reader_thread_count; ++i) {
+  for (i = 0; i < reader_thread_count; ++i) {
     break_pcap_loop(&reader_threads[i]);
   }
 
   printf("------------------------------------ Stopping reader threads\n");
 
-  for (int i = 0; i < reader_thread_count; ++i) {
+  for (i = 0; i < reader_thread_count; ++i) {
     if (reader_threads[i].workflow == NULL) {
       continue;
     }
@@ -1110,7 +1121,7 @@ static int stop_reader_threads(void)
   /* total packets captured: same value for all threads as packet2thread distribution happens later */
   total_packets_captured = reader_threads[0].workflow->packets_captured;
 
-  for (int i = 0; i < reader_thread_count; ++i) {
+  for (i = 0; i < reader_thread_count; ++i) {
     if (reader_threads[i].workflow == NULL) {
       continue;
     }

fuzz/dictionary.dict

@@ -81,6 +81,13 @@
 "RPC_CONNECT"
 "RPC_IN_DATA"
 "RPC_OUT_DATA"
+"MKCOL"
+"MOVE"
+"COPY"
+"LOCK"
+"UNLOCK"
+"PROPFIND"
+"PROPPATCH"
 
 #HTTP payload signatures
 

src/include/ndpi_protocol_ids.h

@@ -403,6 +403,9 @@ typedef enum {
   NDPI_PROTOCOL_HISLIP                = 372,
   NDPI_PROTOCOL_UFTP                  = 373,
   NDPI_PROTOCOL_OPENFLOW              = 374,
+  NDPI_PROTOCOL_JSON_RPC              = 375,
+  NDPI_PROTOCOL_WEBDAV                = 376,
+  NDPI_PROTOCOL_APACHE_KAFKA          = 377,
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"

src/include/ndpi_typedefs.h

@@ -680,6 +680,13 @@ typedef enum {
 	      NDPI_HTTP_METHOD_RPC_CONNECT,
 	      NDPI_HTTP_METHOD_RPC_IN_DATA,
 	      NDPI_HTTP_METHOD_RPC_OUT_DATA,
+	      NDPI_HTTP_METHOD_MKCOL,
+	      NDPI_HTTP_METHOD_MOVE,
+	      NDPI_HTTP_METHOD_COPY,
+	      NDPI_HTTP_METHOD_LOCK,
+	      NDPI_HTTP_METHOD_UNLOCK,
+	      NDPI_HTTP_METHOD_PROPFIND,
+	      NDPI_HTTP_METHOD_PROPPATCH,
 } ndpi_http_method;
 
 typedef enum {

src/lib/ndpi_main.c

@@ -813,6 +813,8 @@ int ndpi_init_empty_app_protocol(ndpi_protocol_match const * const hostname_list
   return 0;
 }
 
+/* ******************************************************************** */
+
 int ndpi_init_app_protocol(struct ndpi_detection_module_struct *ndpi_str,
                            ndpi_protocol_match const * const match) {
   ndpi_port_range ports_a[MAX_DEFAULT_PORTS], ports_b[MAX_DEFAULT_PORTS];
@@ -1056,6 +1058,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			      NDPI_PROTOCOL_MPEGDASH,
 			      NDPI_PROTOCOL_RTSP,
 			      NDPI_PROTOCOL_APACHE_THRIFT,
+			      NDPI_PROTOCOL_JSON_RPC,
 			      NDPI_PROTOCOL_MATCHED_BY_CONTENT,
 			      NDPI_PROTOCOL_NO_MORE_SUBPROTOCOLS); /* NDPI_PROTOCOL_HTTP can have (content-matched) subprotocols */
   ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_MDNS,
@@ -2166,13 +2169,25 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  ndpi_build_default_ports(ports_a, 4880, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_UFTP,
-        "UFTP", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
-        ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-        ndpi_build_default_ports(ports_b, 1044, 0, 0, 0, 0) /* UDP */);
+			  "UFTP", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 1044, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_OPENFLOW,
 			  "OpenFlow", NDPI_PROTOCOL_CATEGORY_NETWORK,
 			  ndpi_build_default_ports(ports_a, 6653, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_JSON_RPC,
+			  "JSON-RPC", NDPI_PROTOCOL_CATEGORY_RPC,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_WEBDAV,
+			  "WebDAV", NDPI_PROTOCOL_CATEGORY_COLLABORATIVE,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0),  /* TCP */
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0)); /* UDP */
+  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_APACHE_KAFKA,
+			  "Kafka", NDPI_PROTOCOL_CATEGORY_RPC,
+			  ndpi_build_default_ports(ports_a, 9092, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main.c"
@@ -5692,6 +5707,12 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
   /* OpenFlow */
   init_openflow_dissector(ndpi_str, &a);
 
+  /* JSON-RPC */
+  init_json_rpc_dissector(ndpi_str, &a);
+
+  /* Apache Kafka */
+  init_kafka_dissector(ndpi_str, &a);
+
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main_init.c"
 #endif
@@ -6949,6 +6970,7 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s
     break;
 
   case NDPI_PROTOCOL_SYSLOG:
+  case NDPI_PROTOCOL_MDNS:
     if(flow->l4_proto == IPPROTO_UDP)
       ndpi_unset_risk(ndpi_str, flow, NDPI_UNIDIRECTIONAL_TRAFFIC);
     break;

src/lib/ndpi_private.h

@@ -610,6 +610,8 @@ void init_profinet_io_dissector(struct ndpi_detection_module_struct *ndpi_struct
 void init_hislip_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id);
 void init_uftp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id);
 void init_openflow_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id);
+void init_json_rpc_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id);
+void init_kafka_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id);
 
 #endif