config: port some configuration knobs to the new API

Remove: `ndpi_set_detection_preferences()`,
`ndpi_set_opportunistic_tls()`, `ndpi_get_opportunistic_tls()`,
`ndpi_set_protocol_aggressiveness()`,
`ndpi_get_protocol_aggressiveness()`

doc/configuration_parameters.md

@@ -6,4 +6,16 @@ TODO
 | Protocol     | Parameter                                 | Default value | Min value | Max value | Description |
 | ------       | ------                                    | ------        | ------    | ------    | ------      |
 | NULL         | "packets_limit_per_flow"                  | 32            | 0         | 255       | The upper limit on the number of packets per flow that will be subject to DPI, after which classification will be considered complete (0 = no limit) |
+| NULL         | "flow.direction_detection.enable"         | 1             | NULL      | NULL      | Enable/disable internal detection of packet direction (client to server or server to client) |
+| NULL         | "flow.track_payload.enable"               | 0             | NULL      | NULL      | Enable/disable tracking/export of flow payload (i.e. L5/7 data) |
+| NULL         | "tcp_ack_payload_heuristic.enable"        | 0             | NULL      | NULL      | In some networks, there are some anomalous TCP flows where the smallest ACK packets have some kind of zero padding. It looks like the IP and TCP headers in those frames wrongly consider the 0x00 Ethernet padding bytes as part of the TCP payload. While this kind of packets is perfectly valid per-se, in some conditions they might be treated by the TCP reassembler logic as (partial) overlaps, deceiving the classification engine. This parameter enable/disable an heuristic to detect these packets and to ignore them, allowing correct detection/classification. See #1946 for other details |
+| NULL         | "fully_encrypted_heuristic.enable"        | 1             | NULL      | NULL      | Enable/disable an heuristic to detect fully encrypted sessions, i.e. flows where every bytes of the payload is encrypted in an attempt to “look like nothing”. This heuristic only analyzes the first packet of the flow. See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf |
+| "tls"        | "certificate_expiration_threshold"        | 30            | 0         | 365       | The threshold (in days) used to trigger the `NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE` flow risk |
+| "tls"        | "application_blocks_tracking.enable"      | 0             | NULL      | NULL      | Enable/disable processing of TLS Application Blocks (post handshake) to extract statistical information about the flow |
 | "tls"        | "metadata.sha1_fingerprint.enable"        | 1             | NULL      | NULL      | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked |
+| "smtp"       | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets in cleartext SMTP flows (because of opportunistic TLS, via STARTTLS msg) |
+| "imap"       | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets in cleartext IMAP flows (because of opportunistic TLS, via STARTTLS msg) |
+| "pop"        | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets in cleartext POP flows (because of opportunistic TLS, via STARTTLS msg) |
+| "ftp"        | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets in cleartext FTP flows (because of opportunistic TLS, via AUTH TLS msg) |
+| "stun"       | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets multiplexed into STUN flows |
+| "ookla"      | "aggressiveness",                         | 1             | 0         | 1         | Detection aggressiveness for Ookla. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable heuristic for detection over TLS (via Ookla LRU cache) |

example/ndpiReader.c

@@ -109,9 +109,8 @@ static int num_cfgs = 0;
 int nDPI_LogLevel = 0;
 char *_debug_protocols = NULL;
 char *_disabled_protocols = NULL;
-int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
 static u_int8_t stats_flag = 0;
-ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic;
+ndpi_init_prefs init_prefs = ndpi_no_prefs;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
 static u_int32_t pcap_analysis_duration = (u_int32_t)-1;
@@ -558,7 +557,6 @@ static void help(u_int long_help) {
          "                            | 2 - List known risks\n"
          "  -d                        | Disable protocol guess and use only DPI\n"
          "  -e <len>                  | Min human readeable string match len. Default %u\n"
-	 "  -E                        | Track flow payload\n"
          "  -q                        | Quiet mode\n"
          "  -F                        | Enable flow stats\n"
          "  -t                        | Dissect GTP/TZSP tunnels\n"
@@ -599,7 +597,6 @@ static void help(u_int long_help) {
          "  -z                        | Enable JA3+\n"
          "  -A                        | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
          "  -M                        | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
-         "  -Z proto:value            | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n"
          "  --lru-cache-size=NAME:size       | Specify the size for this LRU cache (0 to disable it). This flag can be used multiple times\n"
          "  --lru-cache-ttl=NAME:size        | Specify the TTL [in seconds] for this LRU cache (0 to disable it). This flag can be used multiple times\n"
          "  --stun-monitoring=<pkts>:<flags> | Configure STUN monitoring: keep monitoring STUN session for <pkts> more pkts looking for RTP\n"
@@ -1034,16 +1031,13 @@ static void parseOptions(int argc, char **argv) {
   }
 #endif
 
-  for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++)
-    aggressiveness[i] = -1; /* Use the default value */
-
   for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
     lru_cache_sizes[i] = -1; /* Use the default value */
     lru_cache_ttls[i] = -1; /* Use the default value */
   }
 
   while((opt = getopt_long(argc, argv,
-			   "a:Ab:B:e:Ec:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:X:w:zZ:q0123:456:7:89:m:MT:U:",
+			   "a:Ab:B:e:c:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:X:w:zq0123:456:7:89:m:MT:U:",
                            longopts, &option_idx)) != EOF) {
 #ifdef DEBUG_TRACE
     if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
@@ -1075,10 +1069,6 @@ static void parseOptions(int argc, char **argv) {
       human_readeable_string_len = atoi(optarg);
       break;
 
-    case 'E':
-      init_prefs |= ndpi_track_flow_payload;
-      break;
-
     case 'i':
     case '3':
       _pcap_file[0] = optarg;
@@ -1179,36 +1169,6 @@ static void parseOptions(int argc, char **argv) {
       _disabled_protocols = ndpi_strdup(optarg);
       break;
 
-    case 'Z': /* proto_name:aggr_value */
-      {
-        struct ndpi_detection_module_struct *module_tmp;
-        NDPI_PROTOCOL_BITMASK all;
-        char *saveptr, *tmp_str, *proto_str, *aggr_str;
-
-        /* Use a temporary module with all protocols enabled */
-        module_tmp = ndpi_init_detection_module(0);
-        if(!module_tmp)
-          break;
-
-        NDPI_BITMASK_SET_ALL(all);
-        ndpi_set_protocol_detection_bitmask2(module_tmp, &all);
-        ndpi_finalize_initialization(module_tmp);
-
-        tmp_str = ndpi_strdup(optarg);
-        if(tmp_str) {
-          proto_str = strtok_r(tmp_str, ":", &saveptr);
-          if(proto_str) {
-            aggr_str = strtok_r(NULL, ":", &saveptr);
-            if(aggr_str) {
-              aggressiveness[ndpi_get_protocol_id(module_tmp, proto_str)] = atoi(aggr_str);
-            }
-          }
-        }
-        ndpi_free(tmp_str);
-        ndpi_exit_detection_module(module_tmp);
-        break;
-      }
-
     case 'h':
       help(0);
       break;
@@ -2817,12 +2777,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
 			     i, lru_cache_ttls[i]);
   }
 
-  /* Set aggressiviness here */
-  for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
-    if(aggressiveness[i] != -1)
-      ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]);
-  }
-
   for(i = 0; i < num_cfgs; i++) {
     rc = ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct,
 			 cfgs[i].proto, cfgs[i].param, cfgs[i].value);
@@ -2836,10 +2790,12 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
     ndpi_set_monitoring_state(ndpi_thread_info[thread_id].workflow->ndpi_struct, NDPI_PROTOCOL_STUN,
                               stun_monitoring_pkts_to_process, stun_monitoring_flags);
 
-  ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);
-
   if(enable_doh_dot_detection)
-    ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct, ndpi_pref_enable_tls_block_dissection, 1);
+    ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, "tls", "application_blocks_tracking.enable", "1");
+
+  ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1");
+
+  ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);
 }
 
 /* *********************************************** */

fuzz/fuzz_config.cpp

@@ -32,12 +32,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 
   if(fuzzed_data.remaining_bytes() < 4 + /* ndpi_init_detection_module() */
 				     NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS +
-				     1 + /* TLS cert expire */
 				     6 + /* files */
 				     ((NDPI_LRUCACHE_MAX + 1) * 5) + /* LRU caches */
-				     2 + 1 + /* ndpi_set_detection_preferences() */
 				     1 + 3 + 1 + 3 + /* Monitoring */
-				     7 + /* Opportunistic tls */
 				     2 + /* Pid */
 				     2 + /* Category */
 				     1 + /* Tunnel */
@@ -69,8 +66,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 
   /* TODO: ndpi_config_set */
 
-  ndpi_set_tls_cert_expire_days(ndpi_info_mod, fuzzed_data.ConsumeIntegral<u_int8_t>());
-
   if(fuzzed_data.ConsumeBool())
     ndpi_load_protocols_file(ndpi_info_mod, "protos.txt");
   if(fuzzed_data.ConsumeBool())
@@ -98,15 +93,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   /* TODO: stub for geo stuff */
   ndpi_load_geoip(ndpi_info_mod, NULL, NULL);
 
-  if(fuzzed_data.ConsumeBool())
-    ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_direction_detect_disable,
-                                   fuzzed_data.ConsumeBool());
-  if(fuzzed_data.ConsumeBool())
-    ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_enable_tls_block_dissection,
-                                   0 /* unused */);
-
-  ndpi_set_detection_preferences(ndpi_info_mod, static_cast<ndpi_detection_preference>(0xFF), 0xFF); /* Invalid preference */
-
   if(fuzzed_data.ConsumeBool()) {
     ndpi_set_monitoring_state(ndpi_info_mod, NDPI_PROTOCOL_STUN,
                               fuzzed_data.ConsumeIntegralInRange(0, (1 << 16)),
@@ -119,25 +105,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   ndpi_set_monitoring_state(ndpi_info_mod, random_proto, random_value, random_value);
   ndpi_get_monitoring_state(ndpi_info_mod, random_proto, &num, &num2);
 
-  ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_SMTP, fuzzed_data.ConsumeBool());
-  ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_SMTP);
-  ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_IMAP, fuzzed_data.ConsumeBool());
-  ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_IMAP);
-  ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_POP, fuzzed_data.ConsumeBool());
-  ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_POP);
-  ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_FTP_CONTROL, fuzzed_data.ConsumeBool());
-  ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_FTP_CONTROL);
-
-  random_proto = fuzzed_data.ConsumeIntegralInRange(0, (1 << 16) - 1);
-  random_value = fuzzed_data.ConsumeIntegralInRange(0,2); /* Only 0-1 are valid values */
-  ndpi_set_opportunistic_tls(ndpi_info_mod, random_proto, random_value);
-  ndpi_get_opportunistic_tls(ndpi_info_mod, random_proto);
-
-  for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
-    ndpi_set_protocol_aggressiveness(ndpi_info_mod, i, random_value);
-    ndpi_get_protocol_aggressiveness(ndpi_info_mod, i);
-  }
-
   ndpi_finalize_initialization(ndpi_info_mod);
 
   /* Random protocol configuration */

fuzz/fuzz_ndpi_reader.c

@@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 1;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic;
+ndpi_init_prefs init_prefs = ndpi_enable_ja3_plus;
 int enable_malloc_bins = 1;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;
@@ -71,7 +71,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     ndpi_load_malicious_ja3_file(workflow->ndpi_struct, "ja3_fingerprints.csv");
     ndpi_load_malicious_sha1_file(workflow->ndpi_struct, "sha1_fingerprints.csv");
 
-    ndpi_set_detection_preferences(workflow->ndpi_struct, ndpi_pref_enable_tls_block_dissection, 0 /* unused */);
+    ndpi_set_config(workflow->ndpi_struct, NULL, "flow.track_payload.enable", "1");
+    ndpi_set_config(workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1");
+    ndpi_set_config(workflow->ndpi_struct, "tls", "application_blocks_tracking.enable", "1");
 
     ndpi_set_monitoring_state(workflow->ndpi_struct, NDPI_PROTOCOL_STUN,
                               10, NDPI_MONITORING_STUN_SUBCLASSIFIED);

fuzz/fuzz_readerutils_parseprotolist.cpp

@@ -13,7 +13,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 0;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic;
+ndpi_init_prefs init_prefs = ndpi_no_prefs; /* unused */
 int enable_malloc_bins = 0;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;

fuzz/fuzz_readerutils_workflow.cpp

@@ -15,7 +15,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 0;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic;
+ndpi_init_prefs init_prefs = ndpi_enable_ja3_plus;
 int enable_malloc_bins = 0;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;
@@ -93,6 +93,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     NDPI_BITMASK_SET_ALL(enabled_bitmask);
     rc = ndpi_set_protocol_detection_bitmask2(w->ndpi_struct, &enabled_bitmask);
     if(rc == 0) {
+      ndpi_set_config(w->ndpi_struct, NULL, "flow.track_payload.enable", "1");
+      ndpi_set_config(w->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1");
       ndpi_finalize_initialization(w->ndpi_struct);
 
       header = NULL;

src/include/ndpi_api.h

@@ -1036,9 +1036,6 @@ extern "C" {
   int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_struct,
 				     char *name_or_ip, u_int name_len,
 				     ndpi_protocol_category_t *id);
-  int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod,
-				     ndpi_detection_preference pref,
-				     int value);
 
   u_int16_t ndpi_map_user_proto_id_to_ndpi_id(struct ndpi_detection_module_struct *ndpi_str,
 					      u_int16_t user_proto_id);
@@ -1089,16 +1086,6 @@ extern "C" {
 			     lru_cache_type cache_type,
 			     u_int32_t *ttl);
 
-  int ndpi_set_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
-				 u_int16_t proto, int value);
-  int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
-				 u_int16_t proto);
-
-  int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
-                                       u_int16_t proto, u_int32_t value);
-  u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
-                                             u_int16_t proto);
-
   int ndpi_set_monitoring_state(struct ndpi_detection_module_struct *ndpi_struct,
                                 u_int16_t proto, u_int32_t num_pkts, u_int32_t flags);
   int ndpi_get_monitoring_state(struct ndpi_detection_module_struct *ndpi_struct,
@@ -1120,17 +1107,6 @@ extern "C" {
 				    ndpi_protocol_category_t *category,
 				    ndpi_protocol_breed_t *breed);
 
-  /**
-   * Specifies the threshold used to trigger the NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE
-   * flow risk that by default is set to 30 days
-   *
-   * @par    ndpi_struct  = the struct created for the protocol detection
-   * @par    days         = the number of days threshold for emitting the alert
-   *
-   */
-  void ndpi_set_tls_cert_expire_days(struct ndpi_detection_module_struct *ndpi_str,
-				     u_int8_t days);
-
   void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
 				   struct ndpi_flow_struct *flow);