config: remove enum ndpi_prefs

doc/configuration_parameters.md

@@ -7,6 +7,10 @@ TODO
 | ------       | ------                                    | ------        | ------    | ------    | ------      | ------ |
 | NULL         | "packets_limit_per_flow"                  | 32            | 0         | 255       | The upper limit on the number of packets per flow that will be subject to DPI, after which classification will be considered complete (0 = no limit) |
 | NULL         | "flow.direction_detection.enable"         | 1             | NULL      | NULL      | Enable/disable internal detection of packet direction (client to server or server to client) |
+| NULL         | "flow.track_payload.enable"               | 0             | NULL      | NULL      | Enable/disable tracking/export of flow payload (i.e. L5/7 data) |
+| NULL         | "tcp_ack_payload_heuristic.enable"        | 0             | NULL      | NULL      | In some networks, there are some anomalous TCP flows where the smallest ACK packets have some kind of zero padding. It looks like the IP and TCP headers in those frames wrongly consider the 0x00 Ethernet padding bytes as part of the TCP payload. While this kind of packets is perfectly valid per-se, in some conditions they might be treated by the TCP reassembler logic as (partial) overlaps, deceiving the classification engine. This parameter enable/disable an heuristic to detect these packets and to ignore them, allowing correct detection/classification. See #1946 for other details |
+| NULL         | "fully_encrypted_heuristic.enable"        | 1             | NULL      | NULL      | Enable/disable an heuristic to detect fully encrypted sessions, i.e. flows where every bytes of the payload is encrypted in an attempt to “look like nothing”. This heuristic only analyzes the first packet of the flow. See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf |
+| NULL         | "libgcrypt.init"                          | 1             | NULL      | NULL      | Enable/disable initialization of libgcrypt. When using the external libgcrypt (instead of the internal crypto code) the libgcrypt runtime must be initialized. If, for whatever reasons, the application alread does it, nDPI must be told to skip it |
 | NULL         | "flow_risk_lists.load"                    | 1             | NULL      | NULL      | Enable/disable loading of every IP addresses lists used to check any flow risks |
 | NULL         | "flow_risk.anonymous_subscriber.list.icloudprivaterelay.load" | 1             | NULL      | NULL      | Enable/disable loading of internal iCouldPrivateRealy IP address list used to check `NDPI_ANONYMOUS_SUBSCRIBER` flow risk |
 | NULL         | "flow_risk.anonymous_subscriber.list.protonvpn.load"          | 1             | NULL      | NULL      | Enable/disable loading of internal IP address list of ProtonVPN exit nodes used to check `NDPI_ANONYMOUS_SUBSCRIBER` flow risk  |

example/ndpiReader.c

@@ -108,7 +108,6 @@ int nDPI_LogLevel = 0;
 char *_debug_protocols = NULL;
 char *_disabled_protocols = NULL;
 static u_int8_t stats_flag = 0;
-ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
 static u_int32_t pcap_analysis_duration = (u_int32_t)-1;
@@ -331,7 +330,7 @@ void ndpiCheckHostStringMatch(char *testChar) {
   if(!testChar)
     return;
 
-  ndpi_str = ndpi_init_detection_module(init_prefs);
+  ndpi_str = ndpi_init_detection_module();
   ndpi_finalize_initialization(ndpi_str);
 
   testRes =  ndpi_match_string_subprotocol(ndpi_str,
@@ -376,7 +375,7 @@ static void ndpiCheckIPMatch(char *testChar) {
   if(!testChar)
     return;
 
-  ndpi_str = ndpi_init_detection_module(init_prefs);
+  ndpi_str = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_str, &all);
 
@@ -561,7 +560,6 @@ static void help(u_int long_help) {
          "                            | 2 - List known risks\n"
          "  -d                        | Disable protocol guess and use only DPI\n"
          "  -e <len>                  | Min human readeable string match len. Default %u\n"
-	 "  -E                        | Track flow payload\n"
          "  -q                        | Quiet mode\n"
          "  -F                        | Enable flow stats\n"
          "  -t                        | Dissect GTP/TZSP tunnels\n"
@@ -609,7 +607,7 @@ static void help(u_int long_help) {
          max_num_reported_top_payloads, max_num_tcp_dissected_pkts, max_num_udp_dissected_pkts);
 
   NDPI_PROTOCOL_BITMASK all;
-  struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module(init_prefs);
+  struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);
   ndpi_finalize_initialization(ndpi_info_mod);
@@ -769,7 +767,7 @@ void extcap_config() {
   ndpi_proto_defaults_t *proto_defaults;
 #endif
 
-  struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module(init_prefs);
+  struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module();
 #if 0
   ndpi_num_supported_protocols = ndpi_get_ndpi_num_supported_protocols(ndpi_info_mod);
   proto_defaults = ndpi_get_proto_defaults(ndpi_info_mod);
@@ -988,7 +986,7 @@ static void parseOptions(int argc, char **argv) {
 #endif
 
   while((opt = getopt_long(argc, argv,
-			   "a:Ab:B:e:Ec:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:Rs:tu:v:V:n:rp:x:X:w:q0123:456:7:89:m:MT:U:",
+			   "a:Ab:B:e:c:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:Rs:tu:v:V:n:rp:x:X:w:q0123:456:7:89:m:MT:U:",
                            longopts, &option_idx)) != EOF) {
 #ifdef DEBUG_TRACE
     if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
@@ -1020,10 +1018,6 @@ static void parseOptions(int argc, char **argv) {
       human_readeable_string_len = atoi(optarg);
       break;
 
-    case 'E':
-      init_prefs |= ndpi_track_flow_payload;
-      break;
-
     case 'i':
     case '3':
       _pcap_file[0] = optarg;
@@ -1239,7 +1233,7 @@ static void parseOptions(int argc, char **argv) {
 
     case '9':
     {
-      struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module(init_prefs);
+      struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module();
       extcap_packet_filter = ndpi_get_proto_by_name(ndpi_info_mod, optarg);
       if(extcap_packet_filter == NDPI_PROTOCOL_UNKNOWN) extcap_packet_filter = atoi(optarg);
       ndpi_exit_detection_module(ndpi_info_mod);
@@ -2773,6 +2767,8 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
   if(_protoFilePath != NULL)
     ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath);
 
+  ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1");
+
   for(i = 0; i < num_cfgs; i++) {
     rc = ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct,
 			 cfgs[i].proto, cfgs[i].param, cfgs[i].value);
@@ -4756,7 +4752,7 @@ static void dgaUnitTest() {
   };
   int debug = 0, i;
   NDPI_PROTOCOL_BITMASK all;
-  struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(init_prefs);
+  struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module();
 
   assert(ndpi_str != NULL);
 

example/ndpiSimpleIntegration.c

@@ -188,8 +188,7 @@ static struct nDPI_workflow * init_workflow(char const * const file_or_device)
     printf("pcap_setfilter error: '%s'\n", pcap_geterr(workflow->pcap_handle));
   }  
 
-  ndpi_init_prefs init_prefs = ndpi_no_prefs;
-  workflow->ndpi_struct = ndpi_init_detection_module(init_prefs);
+  workflow->ndpi_struct = ndpi_init_detection_module();
   if (workflow->ndpi_struct == NULL) {
     free_workflow(&workflow);
     return NULL;

example/reader_util.c

@@ -79,7 +79,6 @@ extern u_int8_t max_num_udp_dissected_pkts /* 24 */, max_num_tcp_dissected_pkts
 static u_int32_t flow_id = 0;
 
 u_int8_t enable_doh_dot_detection = 0;
-extern ndpi_init_prefs init_prefs;
 
 extern int malloc_size_stats;
 extern struct ndpi_bin malloc_bins;
@@ -410,7 +409,7 @@ int parse_proto_name_list(char *str, NDPI_PROTOCOL_BITMASK *bitmask, int inverte
   else
    op = 0; /* Default action: remove from the bitmask */
   /* Use a temporary module with all protocols enabled */
-  module = ndpi_init_detection_module(0);
+  module = ndpi_init_detection_module();
   if(!module)
     return 1;
   NDPI_BITMASK_SET_ALL(all);
@@ -466,7 +465,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
 #endif
 
   /* TODO: just needed here to init ndpi ndpi_malloc wrapper */
-  module = ndpi_init_detection_module(init_prefs);
+  module = ndpi_init_detection_module();
 
   if(module == NULL) {
     LOG(NDPI_LOG_ERROR, "global structure initialization failed\n");

fuzz/fuzz_common_code.c

@@ -36,12 +36,11 @@ void fuzz_set_alloc_callbacks_and_seed(int seed)
 
 void fuzz_init_detection_module(struct ndpi_detection_module_struct **ndpi_info_mod)
 {
-  ndpi_init_prefs prefs = ndpi_no_prefs;
   NDPI_PROTOCOL_BITMASK all;
   NDPI_PROTOCOL_BITMASK debug_bitmask;
 
   if(*ndpi_info_mod == NULL) {
-    *ndpi_info_mod = ndpi_init_detection_module(prefs);
+    *ndpi_info_mod = ndpi_init_detection_module();
 
     NDPI_BITMASK_SET_ALL(debug_bitmask);
     ndpi_set_log_level(*ndpi_info_mod, 4);

fuzz/fuzz_config.cpp

@@ -39,7 +39,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  ndpi_info_mod = ndpi_init_detection_module(fuzzed_data.ConsumeIntegral<u_int32_t>());
+  ndpi_info_mod = ndpi_init_detection_module();
 
   set_ndpi_debug_function(ndpi_info_mod, NULL);
 

fuzz/fuzz_filecfg_categories.c

@@ -11,7 +11,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  ndpi_struct = ndpi_init_detection_module(ndpi_no_prefs);
+  ndpi_struct = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
 

fuzz/fuzz_filecfg_malicious_ja3.c

@@ -11,7 +11,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  ndpi_struct = ndpi_init_detection_module(ndpi_no_prefs);
+  ndpi_struct = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
 

fuzz/fuzz_filecfg_malicious_sha1.c

@@ -11,7 +11,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  ndpi_struct = ndpi_init_detection_module(ndpi_no_prefs);
+  ndpi_struct = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
 

fuzz/fuzz_filecfg_protocols.c

@@ -11,7 +11,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  ndpi_struct = ndpi_init_detection_module(ndpi_no_prefs);
+  ndpi_struct = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
 

fuzz/fuzz_filecfg_risk_domains.c

@@ -11,7 +11,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* To allow memory allocation failures */
   fuzz_set_alloc_callbacks_and_seed(size);
 
-  ndpi_struct = ndpi_init_detection_module(ndpi_no_prefs);
+  ndpi_struct = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
 

fuzz/fuzz_ndpi_reader.c

@@ -18,7 +18,6 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 1;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_tcp_ack_payload_heuristic;
 int enable_malloc_bins = 1;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;
@@ -72,6 +71,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     NDPI_BITMASK_SET_ALL(all);
     ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
 
+    ndpi_set_config(workflow->ndpi_struct, NULL, "flow.track_payload.enable", "1");
+    ndpi_set_config(workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1");
     ndpi_set_config(workflow->ndpi_struct, "tls", "application_blocks_tracking.enable", "1");
 
     memset(workflow->stats.protocol_counter, 0,

fuzz/fuzz_readerutils_parseprotolist.cpp

@@ -13,7 +13,6 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 0;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_tcp_ack_payload_heuristic;
 int enable_malloc_bins = 0;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;

fuzz/fuzz_readerutils_workflow.cpp

@@ -15,7 +15,6 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 0;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_tcp_ack_payload_heuristic;
 int enable_malloc_bins = 0;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;

python/ndpi/ndpi.py

@@ -32,7 +32,7 @@ class NDPI(object):
                  "_detection_module")
 
     def __init__(self):
-        self._detection_module = lib.ndpi_init_detection_module(0)
+        self._detection_module = lib.ndpi_init_detection_module()
         if self._detection_module == ffi.NULL:
             raise MemoryError("Unable to instantiate NDPI object")
         lib.ndpi_py_setup_detection_module(self._detection_module)

python/ndpi/ndpi_build.py

@@ -49,7 +49,7 @@
 NDPI_APIS = """
 u_int16_t ndpi_get_api_version(void);
 char* ndpi_revision(void);
-struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs prefs);
+struct ndpi_detection_module_struct *ndpi_init_detection_module();
 void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct);
 void ndpi_flow_free(void *ptr);
 ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,

src/include/ndpi_api.h

@@ -211,11 +211,10 @@ extern "C" {
    * indipendent detection contexts) but all these calls MUST NOT run
    * in parallel
    *
-   * @par prefs = load preferences
    * @return  the initialized detection module
    *
    */
-  struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs prefs);
+  struct ndpi_detection_module_struct *ndpi_init_detection_module(void);
 
   /**
    * Completes the initialization (2nd step)

src/include/ndpi_typedefs.h

@@ -1505,30 +1505,6 @@ typedef struct {
   u_int16_t value;
 } ndpi_network6;
 
-typedef u_int32_t ndpi_init_prefs;
-
-typedef enum {
-    ndpi_no_prefs                  = 0,
-    ndpi_dont_init_libgcrypt       = (1 << 1),
-    ndpi_track_flow_payload        = (1 << 16),
-    /* In some networks, there are some anomalous TCP flows where
-       the smallest ACK packets have some kind of zero padding.
-       It looks like the IP and TCP headers in those frames wrongly consider the
-       0x00 Ethernet padding bytes as part of the TCP payload.
-       While this kind of packets is perfectly valid per-se, in some conditions
-       they might be treated by the TCP reassembler logic as (partial) overlaps,
-       deceiving the classification engine.
-       Add an heuristic to detect these packets and to ignore them, allowing
-       correct detection/classification.
-       See #1946 for other details */
-    ndpi_enable_tcp_ack_payload_heuristic = (1 << 17),
-    /* Heuristic to detect fully encrypted sessions, i.e. flows where every bytes of
-       the payload is encrypted in an attempt to “look like nothing”.
-       This heuristic only analyzes the first packet of the flow.
-       See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf */
-    ndpi_disable_fully_encrypted_heuristic = (1 << 20),
-  } ndpi_prefs;
-
 typedef struct {
   u_int32_t protocol_id;
   ndpi_protocol_category_t protocol_category;