diff --git a/doc/configuration_parameters.md b/doc/configuration_parameters.md index b55d997a58f1..9c418c83e1a7 100644 --- a/doc/configuration_parameters.md +++ b/doc/configuration_parameters.md @@ -6,4 +6,16 @@ TODO | Protocol | Parameter | Default value | Min value | Max value | Description | | ------ | ------ | ------ | ------ | ------ | ------ | | NULL | "packets_limit_per_flow" | 32 | 0 | 255 | The upper limit on the number of packets per flow that will be subject to DPI, after which classification will be considered complete (0 = no limit) | +| NULL | "flow.direction_detection.enable" | 1 | NULL | NULL | Enable/disable internal detection of packet direction (client to server or server to client) | +| NULL | "flow.track_payload.enable" | 0 | NULL | NULL | Enable/disable tracking/export of flow payload (i.e. L5/7 data) | +| NULL | "tcp_ack_payload_heuristic.enable" | 0 | NULL | NULL | In some networks, there are some anomalous TCP flows where the smallest ACK packets have some kind of zero padding. It looks like the IP and TCP headers in those frames wrongly consider the 0x00 Ethernet padding bytes as part of the TCP payload. While this kind of packets is perfectly valid per-se, in some conditions they might be treated by the TCP reassembler logic as (partial) overlaps, deceiving the classification engine. This parameter enable/disable an heuristic to detect these packets and to ignore them, allowing correct detection/classification. See #1946 for other details | +| NULL | "fully_encrypted_heuristic.enable" | 1 | NULL | NULL | Enable/disable an heuristic to detect fully encrypted sessions, i.e. flows where every bytes of the payload is encrypted in an attempt to “look like nothing”. This heuristic only analyzes the first packet of the flow. See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf | +| "tls" | "certificate_expiration_threshold" | 30 | 0 | 365 | The threshold (in days) used to trigger the `NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE` flow risk | +| "tls" | "application_blocks_tracking.enable" | 0 | NULL | NULL | Enable/disable processing of TLS Application Blocks (post handshake) to extract statistical information about the flow | | "tls" | "metadata.sha1_fingerprint.enable" | 1 | NULL | NULL | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked | +| "smtp" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext SMTP flows (because of opportunistic TLS, via STARTTLS msg) | +| "imap" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext IMAP flows (because of opportunistic TLS, via STARTTLS msg) | +| "pop" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext POP flows (because of opportunistic TLS, via STARTTLS msg) | +| "ftp" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext FTP flows (because of opportunistic TLS, via AUTH TLS msg) | +| "stun" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets multiplexed into STUN flows | +| "ookla" | "aggressiveness", | 1 | 0 | 1 | Detection aggressiveness for Ookla. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable heuristic for detection over TLS (via Ookla LRU cache) | diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 5a19e919aa4f..dfbdd3a69aa6 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -109,9 +109,8 @@ static int num_cfgs = 0; int nDPI_LogLevel = 0; char *_debug_protocols = NULL; char *_disabled_protocols = NULL; -int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS]; static u_int8_t stats_flag = 0; -ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_no_prefs; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; static u_int32_t pcap_analysis_duration = (u_int32_t)-1; @@ -558,7 +557,6 @@ static void help(u_int long_help) { " | 2 - List known risks\n" " -d | Disable protocol guess and use only DPI\n" " -e | Min human readeable string match len. Default %u\n" - " -E | Track flow payload\n" " -q | Quiet mode\n" " -F | Enable flow stats\n" " -t | Dissect GTP/TZSP tunnels\n" @@ -599,7 +597,6 @@ static void help(u_int long_help) { " -z | Enable JA3+\n" " -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n" " -M | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n" - " -Z proto:value | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n" " --lru-cache-size=NAME:size | Specify the size for this LRU cache (0 to disable it). This flag can be used multiple times\n" " --lru-cache-ttl=NAME:size | Specify the TTL [in seconds] for this LRU cache (0 to disable it). This flag can be used multiple times\n" " --stun-monitoring=: | Configure STUN monitoring: keep monitoring STUN session for more pkts looking for RTP\n" @@ -1034,16 +1031,13 @@ static void parseOptions(int argc, char **argv) { } #endif - for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) - aggressiveness[i] = -1; /* Use the default value */ - for(i = 0; i < NDPI_LRUCACHE_MAX; i++) { lru_cache_sizes[i] = -1; /* Use the default value */ lru_cache_ttls[i] = -1; /* Use the default value */ } while((opt = getopt_long(argc, argv, - "a:Ab:B:e:Ec:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:X:w:zZ:q0123:456:7:89:m:MT:U:", + "a:Ab:B:e:c:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:X:w:zq0123:456:7:89:m:MT:U:", longopts, &option_idx)) != EOF) { #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : ""); @@ -1075,10 +1069,6 @@ static void parseOptions(int argc, char **argv) { human_readeable_string_len = atoi(optarg); break; - case 'E': - init_prefs |= ndpi_track_flow_payload; - break; - case 'i': case '3': _pcap_file[0] = optarg; @@ -1179,36 +1169,6 @@ static void parseOptions(int argc, char **argv) { _disabled_protocols = ndpi_strdup(optarg); break; - case 'Z': /* proto_name:aggr_value */ - { - struct ndpi_detection_module_struct *module_tmp; - NDPI_PROTOCOL_BITMASK all; - char *saveptr, *tmp_str, *proto_str, *aggr_str; - - /* Use a temporary module with all protocols enabled */ - module_tmp = ndpi_init_detection_module(0); - if(!module_tmp) - break; - - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(module_tmp, &all); - ndpi_finalize_initialization(module_tmp); - - tmp_str = ndpi_strdup(optarg); - if(tmp_str) { - proto_str = strtok_r(tmp_str, ":", &saveptr); - if(proto_str) { - aggr_str = strtok_r(NULL, ":", &saveptr); - if(aggr_str) { - aggressiveness[ndpi_get_protocol_id(module_tmp, proto_str)] = atoi(aggr_str); - } - } - } - ndpi_free(tmp_str); - ndpi_exit_detection_module(module_tmp); - break; - } - case 'h': help(0); break; @@ -2817,12 +2777,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { i, lru_cache_ttls[i]); } - /* Set aggressiviness here */ - for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) { - if(aggressiveness[i] != -1) - ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]); - } - for(i = 0; i < num_cfgs; i++) { rc = ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, cfgs[i].proto, cfgs[i].param, cfgs[i].value); @@ -2836,10 +2790,12 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { ndpi_set_monitoring_state(ndpi_thread_info[thread_id].workflow->ndpi_struct, NDPI_PROTOCOL_STUN, stun_monitoring_pkts_to_process, stun_monitoring_flags); - ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct); - if(enable_doh_dot_detection) - ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct, ndpi_pref_enable_tls_block_dissection, 1); + ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, "tls", "application_blocks_tracking.enable", "1"); + + ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1"); + + ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct); } /* *********************************************** */ diff --git a/fuzz/fuzz_config.cpp b/fuzz/fuzz_config.cpp index eea0ec0677cf..dcd738c10198 100644 --- a/fuzz/fuzz_config.cpp +++ b/fuzz/fuzz_config.cpp @@ -32,12 +32,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { if(fuzzed_data.remaining_bytes() < 4 + /* ndpi_init_detection_module() */ NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS + - 1 + /* TLS cert expire */ 6 + /* files */ ((NDPI_LRUCACHE_MAX + 1) * 5) + /* LRU caches */ - 2 + 1 + /* ndpi_set_detection_preferences() */ 1 + 3 + 1 + 3 + /* Monitoring */ - 7 + /* Opportunistic tls */ 2 + /* Pid */ 2 + /* Category */ 1 + /* Tunnel */ @@ -69,8 +66,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { /* TODO: ndpi_config_set */ - ndpi_set_tls_cert_expire_days(ndpi_info_mod, fuzzed_data.ConsumeIntegral()); - if(fuzzed_data.ConsumeBool()) ndpi_load_protocols_file(ndpi_info_mod, "protos.txt"); if(fuzzed_data.ConsumeBool()) @@ -98,15 +93,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { /* TODO: stub for geo stuff */ ndpi_load_geoip(ndpi_info_mod, NULL, NULL); - if(fuzzed_data.ConsumeBool()) - ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_direction_detect_disable, - fuzzed_data.ConsumeBool()); - if(fuzzed_data.ConsumeBool()) - ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_enable_tls_block_dissection, - 0 /* unused */); - - ndpi_set_detection_preferences(ndpi_info_mod, static_cast(0xFF), 0xFF); /* Invalid preference */ - if(fuzzed_data.ConsumeBool()) { ndpi_set_monitoring_state(ndpi_info_mod, NDPI_PROTOCOL_STUN, fuzzed_data.ConsumeIntegralInRange(0, (1 << 16)), @@ -119,25 +105,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { ndpi_set_monitoring_state(ndpi_info_mod, random_proto, random_value, random_value); ndpi_get_monitoring_state(ndpi_info_mod, random_proto, &num, &num2); - ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_SMTP, fuzzed_data.ConsumeBool()); - ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_SMTP); - ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_IMAP, fuzzed_data.ConsumeBool()); - ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_IMAP); - ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_POP, fuzzed_data.ConsumeBool()); - ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_MAIL_POP); - ndpi_set_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_FTP_CONTROL, fuzzed_data.ConsumeBool()); - ndpi_get_opportunistic_tls(ndpi_info_mod, NDPI_PROTOCOL_FTP_CONTROL); - - random_proto = fuzzed_data.ConsumeIntegralInRange(0, (1 << 16) - 1); - random_value = fuzzed_data.ConsumeIntegralInRange(0,2); /* Only 0-1 are valid values */ - ndpi_set_opportunistic_tls(ndpi_info_mod, random_proto, random_value); - ndpi_get_opportunistic_tls(ndpi_info_mod, random_proto); - - for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) { - ndpi_set_protocol_aggressiveness(ndpi_info_mod, i, random_value); - ndpi_get_protocol_aggressiveness(ndpi_info_mod, i); - } - ndpi_finalize_initialization(ndpi_info_mod); /* Random protocol configuration */ diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c index 1a6cd7569e78..f749eef7c59f 100644 --- a/fuzz/fuzz_ndpi_reader.c +++ b/fuzz/fuzz_ndpi_reader.c @@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 1; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_enable_ja3_plus; int enable_malloc_bins = 1; int malloc_size_stats = 0; int max_malloc_bins = 14; @@ -71,7 +71,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { ndpi_load_malicious_ja3_file(workflow->ndpi_struct, "ja3_fingerprints.csv"); ndpi_load_malicious_sha1_file(workflow->ndpi_struct, "sha1_fingerprints.csv"); - ndpi_set_detection_preferences(workflow->ndpi_struct, ndpi_pref_enable_tls_block_dissection, 0 /* unused */); + ndpi_set_config(workflow->ndpi_struct, NULL, "flow.track_payload.enable", "1"); + ndpi_set_config(workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1"); + ndpi_set_config(workflow->ndpi_struct, "tls", "application_blocks_tracking.enable", "1"); ndpi_set_monitoring_state(workflow->ndpi_struct, NDPI_PROTOCOL_STUN, 10, NDPI_MONITORING_STUN_SUBCLASSIFIED); diff --git a/fuzz/fuzz_readerutils_parseprotolist.cpp b/fuzz/fuzz_readerutils_parseprotolist.cpp index 771719a8ff3f..44e195b73c99 100644 --- a/fuzz/fuzz_readerutils_parseprotolist.cpp +++ b/fuzz/fuzz_readerutils_parseprotolist.cpp @@ -13,7 +13,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 0; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_no_prefs; /* unused */ int enable_malloc_bins = 0; int malloc_size_stats = 0; int max_malloc_bins = 14; diff --git a/fuzz/fuzz_readerutils_workflow.cpp b/fuzz/fuzz_readerutils_workflow.cpp index f753560f271d..d5fef8fddf91 100644 --- a/fuzz/fuzz_readerutils_workflow.cpp +++ b/fuzz/fuzz_readerutils_workflow.cpp @@ -15,7 +15,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 0; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_enable_ja3_plus; int enable_malloc_bins = 0; int malloc_size_stats = 0; int max_malloc_bins = 14; @@ -93,6 +93,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { NDPI_BITMASK_SET_ALL(enabled_bitmask); rc = ndpi_set_protocol_detection_bitmask2(w->ndpi_struct, &enabled_bitmask); if(rc == 0) { + ndpi_set_config(w->ndpi_struct, NULL, "flow.track_payload.enable", "1"); + ndpi_set_config(w->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1"); ndpi_finalize_initialization(w->ndpi_struct); header = NULL; diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 5d70bed5bbf0..ead148aae88f 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -1036,9 +1036,6 @@ extern "C" { int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_struct, char *name_or_ip, u_int name_len, ndpi_protocol_category_t *id); - int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod, - ndpi_detection_preference pref, - int value); u_int16_t ndpi_map_user_proto_id_to_ndpi_id(struct ndpi_detection_module_struct *ndpi_str, u_int16_t user_proto_id); @@ -1089,16 +1086,6 @@ extern "C" { lru_cache_type cache_type, u_int32_t *ttl); - int ndpi_set_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto, int value); - int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto); - - int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto, u_int32_t value); - u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto); - int ndpi_set_monitoring_state(struct ndpi_detection_module_struct *ndpi_struct, u_int16_t proto, u_int32_t num_pkts, u_int32_t flags); int ndpi_get_monitoring_state(struct ndpi_detection_module_struct *ndpi_struct, @@ -1120,17 +1107,6 @@ extern "C" { ndpi_protocol_category_t *category, ndpi_protocol_breed_t *breed); - /** - * Specifies the threshold used to trigger the NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE - * flow risk that by default is set to 30 days - * - * @par ndpi_struct = the struct created for the protocol detection - * @par days = the number of days threshold for emitting the alert - * - */ - void ndpi_set_tls_cert_expire_days(struct ndpi_detection_module_struct *ndpi_str, - u_int8_t days); - void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow); diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index f978b314dae0..9a0b947daa5a 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -745,15 +745,6 @@ struct ndpi_lru_cache { struct ndpi_lru_cache_entry *entries; }; - -/* Aggressiveness values */ - -#define NDPI_AGGRESSIVENESS_DISABLED 0x00 /* For all protocols */ - -/* Ookla */ -#define NDPI_AGGRESSIVENESS_OOKLA_TLS 0x01 /* Enable detection over TLS (using ookla cache) */ - - /* Monitoring flags */ /* Stun */ @@ -1066,11 +1057,6 @@ typedef enum { NDPI_PROTOCOL_ANY_CATEGORY /* Used to handle wildcards */ } ndpi_protocol_category_t; -typedef enum { - ndpi_pref_direction_detect_disable = 0, - ndpi_pref_enable_tls_block_dissection, /* nDPI considers only those blocks past the certificate exchange */ -} ndpi_detection_preference; - /* ntop extensions */ typedef struct ndpi_proto_defaults { char *protoName; @@ -1155,10 +1141,41 @@ struct ndpi_risk_information { struct ndpi_detection_module_config_struct { int max_packets_to_process; + int direction_detect_enabled; + /* In some networks, there are some anomalous TCP flows where + the smallest ACK packets have some kind of zero padding. + It looks like the IP and TCP headers in those frames wrongly consider the + 0x00 Ethernet padding bytes as part of the TCP payload. + While this kind of packets is perfectly valid per-se, in some conditions + they might be treated by the TCP reassembler logic as (partial) overlaps, + deceiving the classification engine. + Add an heuristic to detect these packets and to ignore them, allowing + correct detection/classification. + See #1946 for other details */ + int tcp_ack_paylod_heuristic; + /* Heuristic to detect fully encrypted sessions, i.e. flows where every bytes of + the payload is encrypted in an attempt to “look like nothing”. + This heuristic only analyzes the first packet of the flow. + See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf */ + int fully_encrypted_heuristic; + int track_payload_enabled; /* TLS */ + int tls_certificate_expire_in_x_days; + int tls_app_blocks_tracking_enabled; int tls_sha1_fingerprint_enabled; + int smtp_opportunistic_tls_enabled; + + int imap_opportunistic_tls_enabled; + + int pop_opportunistic_tls_enabled; + + int ftp_opportunistic_tls_enabled; + + int stun_opportunistic_tls_enabled; + + int ookla_aggressiveness; }; struct ndpi_flow_struct { @@ -1532,25 +1549,8 @@ typedef enum { ndpi_dont_load_icloud_private_relay_list = (1 << 13), ndpi_dont_init_risk_ptree = (1 << 14), ndpi_dont_load_cachefly_list = (1 << 15), - ndpi_track_flow_payload = (1 << 16), - /* In some networks, there are some anomalous TCP flows where - the smallest ACK packets have some kind of zero padding. - It looks like the IP and TCP headers in those frames wrongly consider the - 0x00 Ethernet padding bytes as part of the TCP payload. - While this kind of packets is perfectly valid per-se, in some conditions - they might be treated by the TCP reassembler logic as (partial) overlaps, - deceiving the classification engine. - Add an heuristic to detect these packets and to ignore them, allowing - correct detection/classification. - See #1946 for other details */ - ndpi_enable_tcp_ack_payload_heuristic = (1 << 17), ndpi_dont_load_crawlers_list = (1 << 18), ndpi_dont_load_protonvpn_list = (1 << 19), - /* Heuristic to detect fully encrypted sessions, i.e. flows where every bytes of - the payload is encrypted in an attempt to “look like nothing”. - This heuristic only analyzes the first packet of the flow. - See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf */ - ndpi_disable_fully_encrypted_heuristic = (1 << 20), ndpi_dont_load_protonvpn_exit_nodes_list = (1 << 21), ndpi_dont_load_mullvad_list = (1 << 22), } ndpi_prefs; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 52b92be26a8d..1c9fac58c798 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -958,34 +958,6 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp /* ******************************************************************** */ -int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_str, ndpi_detection_preference pref, - int value) { - if(!ndpi_str) - return -1; - - switch(pref) { - case ndpi_pref_direction_detect_disable: - ndpi_str->direction_detect_disable = (u_int8_t) value; - break; - - case ndpi_pref_enable_tls_block_dissection: - /* - If this option is enabled only the TLS Application data blocks past the - certificate negotiation are considered - */ - ndpi_str->num_tls_blocks_to_follow = NDPI_MAX_NUM_TLS_APPL_BLOCKS; - ndpi_str->skip_tls_blocks_until_change_cipher = 1; - break; - - default: - return(-1); - } - - return(0); -} - -/* ******************************************************************** */ - static void ndpi_validate_protocol_initialization(struct ndpi_detection_module_struct *ndpi_str) { u_int i; @@ -3209,9 +3181,6 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_init_ptree_ipv6(ndpi_str, ndpi_str->protocols_ptree6, ndpi_protocol_roblox_protocol_list_6); } - if(prefs & ndpi_track_flow_payload) - ndpi_str->max_payload_track_len = 1024; /* track up to X payload bytes */ - ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */); ndpi_str->ip_risk_mask_ptree6 = ndpi_patricia_new(128 /* IPv6 */); @@ -3243,7 +3212,6 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_str->user_data = NULL; ndpi_str->tcp_max_retransmission_window_size = NDPI_DEFAULT_MAX_TCP_RETRANSMISSION_WINDOW_SIZE; - ndpi_str->tls_certificate_expire_in_x_days = 30; /* NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE flow risk */ ndpi_str->ndpi_num_supported_protocols = NDPI_MAX_SUPPORTED_PROTOCOLS; ndpi_str->ndpi_num_custom_protocols = 0; @@ -3372,23 +3340,9 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_str->msteams_cache_ttl = 60; /* sec */ ndpi_str->stun_zoom_cache_ttl = 60; /* sec */ - ndpi_str->opportunistic_tls_smtp_enabled = 1; - ndpi_str->opportunistic_tls_imap_enabled = 1; - ndpi_str->opportunistic_tls_pop_enabled = 1; - ndpi_str->opportunistic_tls_ftp_enabled = 1; - ndpi_str->opportunistic_tls_stun_enabled = 1; - ndpi_str->monitoring_stun_pkts_to_process = 4; ndpi_str->monitoring_stun_flags = 0; - ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS; - - if(prefs & ndpi_enable_tcp_ack_payload_heuristic) - ndpi_str->tcp_ack_paylod_heuristic = 1; - - if(!(prefs & ndpi_disable_fully_encrypted_heuristic)) - ndpi_str->fully_encrypted_based_on_first_pkt_heuristic = 1; - for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++) ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u", (unsigned int) (i + 1)); @@ -3533,6 +3487,14 @@ void ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str) } ndpi_str->ac_automa_finalized = 1; + + if(ndpi_str->cfg.tls_app_blocks_tracking_enabled) { + ndpi_str->num_tls_blocks_to_follow = NDPI_MAX_NUM_TLS_APPL_BLOCKS; + ndpi_str->skip_tls_blocks_until_change_cipher = 1; + } + + if(ndpi_str->cfg.track_payload_enabled) + ndpi_str->max_payload_track_len = 1024; /* track up to X payload bytes */ } /* *********************************************** */ @@ -6339,7 +6301,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, packet->tcp_retransmission = 0, packet->packet_direction = 0; - if(ndpi_str->direction_detect_disable) { + if(!ndpi_str->cfg.direction_detect_enabled) { packet->packet_direction = flow->packet_direction; } else { if(iph != NULL && ntohl(iph->saddr) < ntohl(iph->daddr)) @@ -6364,7 +6326,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, else if(flags == (TH_FIN | TH_PUSH | TH_URG)) ndpi_set_risk(ndpi_str, flow, NDPI_TCP_ISSUES, "TCP XMAS scan"); - if(!ndpi_str->direction_detect_disable && + if(ndpi_str->cfg.direction_detect_enabled && (tcph->source != tcph->dest)) packet->packet_direction = (ntohs(tcph->source) < ntohs(tcph->dest)) ? 1 : 0; @@ -6396,7 +6358,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, } } - if(ndpi_str->tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) { + if(ndpi_str->cfg.tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) { NDPI_LOG_DBG2(ndpi_str, "TCP ACK with zero padding. Ignoring\n"); packet->tcp_retransmission = 1; } else if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 || @@ -6445,7 +6407,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str, flow->next_tcp_seq_nr[1] = 0; } } else if(udph != NULL) { - if(!ndpi_str->direction_detect_disable && + if(ndpi_str->cfg.direction_detect_enabled && (udph->source != udph->dest)) packet->packet_direction = (htons(udph->source) < htons(udph->dest)) ? 1 : 0; } @@ -8005,7 +7967,7 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio && (flow->l4_proto == IPPROTO_TCP)) ndpi_add_connection_as_zoom(ndpi_str, flow); - if(ndpi_str->fully_encrypted_based_on_first_pkt_heuristic && + if(ndpi_str->cfg.fully_encrypted_heuristic && ret.app_protocol == NDPI_PROTOCOL_UNKNOWN && /* Only for unknown traffic */ flow->packet_counter == 1 && packet->payload_packet_len > 0) { flow->first_pkt_fully_encrypted = fully_enc_heuristic(ndpi_str, flow); @@ -10451,92 +10413,6 @@ int ndpi_get_monitoring_state(struct ndpi_detection_module_struct *ndpi_struct, /* ******************************************************************** */ -int ndpi_set_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto, int value) -{ - if(!ndpi_struct || (value != 0 && value != 1)) - return -1; - - switch(proto) { - case NDPI_PROTOCOL_MAIL_SMTP: - ndpi_struct->opportunistic_tls_smtp_enabled = value; - return 0; - case NDPI_PROTOCOL_MAIL_IMAP: - ndpi_struct->opportunistic_tls_imap_enabled = value; - return 0; - case NDPI_PROTOCOL_MAIL_POP: - ndpi_struct->opportunistic_tls_pop_enabled = value; - return 0; - case NDPI_PROTOCOL_FTP_CONTROL: - ndpi_struct->opportunistic_tls_ftp_enabled = value; - return 0; - case NDPI_PROTOCOL_STUN: - ndpi_struct->opportunistic_tls_stun_enabled = value; - return 0; - default: - return -1; - } -} - -/* ******************************************************************** */ - -int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto) -{ - if(!ndpi_struct) - return -1; - - switch(proto) { - case NDPI_PROTOCOL_MAIL_SMTP: - return ndpi_struct->opportunistic_tls_smtp_enabled; - case NDPI_PROTOCOL_MAIL_IMAP: - return ndpi_struct->opportunistic_tls_imap_enabled; - case NDPI_PROTOCOL_MAIL_POP: - return ndpi_struct->opportunistic_tls_pop_enabled; - case NDPI_PROTOCOL_FTP_CONTROL: - return ndpi_struct->opportunistic_tls_ftp_enabled; - case NDPI_PROTOCOL_STUN: - return ndpi_struct->opportunistic_tls_stun_enabled; - default: - return -1; - } -} - -/* ******************************************************************** */ - -int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto, u_int32_t value) -{ - if(!ndpi_struct) - return -1; - - switch(proto) { - case NDPI_PROTOCOL_OOKLA: - ndpi_struct->aggressiveness_ookla = value; - return 0; - default: - return -1; - } -} - -/* ******************************************************************** */ - -u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct, - u_int16_t proto) -{ - if(!ndpi_struct) - return -1; - - switch(proto) { - case NDPI_PROTOCOL_OOKLA: - return ndpi_struct->aggressiveness_ookla; - default: - return -1; - } -} - -/* ******************************************************************** */ - void ndpi_set_user_data(struct ndpi_detection_module_struct *ndpi_str, void *user_data) { if (ndpi_str == NULL) @@ -10585,7 +10461,7 @@ static int _set_param_int(void *_variable, const char *value, const char *min_va long val; errno = 0; /* To distinguish success/failure after call */ - val = strtol(value, &endptr, 10); + val = strtol(value, &endptr, 0); /* Check for various possible errors */ if((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || @@ -10639,13 +10515,32 @@ static const struct cfg_param { } cfg_params[] = { /* Per-protocol parameters */ + { "tls", "certificate_expiration_threshold", "30", "0", "365", CFG_PARAM_INT, __OFF(tls_certificate_expire_in_x_days) }, + { "tls", "application_blocks_tracking.enable", "0", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_app_blocks_tracking_enabled) }, /* An example of metadata configuration (yes/no) */ { "tls", "metadata.sha1_fingerprint.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_sha1_fingerprint_enabled) }, + { "smtp", "tls_dissection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(smtp_opportunistic_tls_enabled) }, + + { "imap", "tls_dissection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(imap_opportunistic_tls_enabled) }, + + { "pop", "tls_dissection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(pop_opportunistic_tls_enabled) }, + + { "ftp", "tls_dissection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(ftp_opportunistic_tls_enabled) }, + + { "stun", "tls_dissection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(stun_opportunistic_tls_enabled) }, + + { "ookla", "aggressiveness", "0x01", "0", "1", CFG_PARAM_INT, __OFF(ookla_aggressiveness) }, + + /* Global parameters */ /* An example of integer configuration */ { NULL, "packets_limit_per_flow", "32", "0", "255", CFG_PARAM_INT, __OFF(max_packets_to_process) }, + { NULL, "flow.direction_detection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(direction_detect_enabled) }, + { NULL, "flow.track_payload.enable", "0", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(track_payload_enabled) }, + { NULL, "tcp_ack_payload_heuristic.enable", "0", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tcp_ack_paylod_heuristic) }, + { NULL, "fully_encrypted_heuristic.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(fully_encrypted_heuristic) }, { NULL, NULL, NULL, NULL, NULL, 0, -1 }, }; diff --git a/src/lib/ndpi_private.h b/src/lib/ndpi_private.h index 938bef5e6891..1108adb954b6 100644 --- a/src/lib/ndpi_private.h +++ b/src/lib/ndpi_private.h @@ -128,7 +128,6 @@ struct ndpi_detection_module_struct { u_int64_t current_ts; u_int16_t num_tls_blocks_to_follow; u_int8_t skip_tls_blocks_until_change_cipher:1, enable_ja3_plus:1, _notused:6; - u_int8_t tls_certificate_expire_in_x_days; void *user_data; char custom_category_labels[NUM_CUSTOM_CATEGORIES][CUSTOM_CATEGORY_LABEL_LEN]; @@ -248,25 +247,12 @@ struct ndpi_detection_module_struct { /* *** If you add a new LRU cache, please update lru_cache_type above! *** */ - int opportunistic_tls_smtp_enabled; - int opportunistic_tls_imap_enabled; - int opportunistic_tls_pop_enabled; - int opportunistic_tls_ftp_enabled; - int opportunistic_tls_stun_enabled; - u_int32_t monitoring_stun_pkts_to_process; u_int32_t monitoring_stun_flags; - u_int32_t aggressiveness_ookla; - - int tcp_ack_paylod_heuristic; - int fully_encrypted_based_on_first_pkt_heuristic; - u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */ ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; - u_int8_t direction_detect_disable:1, /* disable internal detection of packet direction */ _pad:7; - #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_typedefs.h" #endif @@ -293,6 +279,11 @@ struct ndpi_detection_module_struct { #define NDPI_PROTOCOL_MATCHED_BY_CONTENT (-2) +/* Aggressiveness values */ + +/* Ookla */ +#define AGGRESSIVENESS_OOKLA_TLS 0x01 /* Enable detection over TLS (using ookla cache) */ + /* Generic */ diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index d6f5b7f60861..2907dcdd1be6 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -2766,14 +2766,6 @@ u_int8_t ndpi_is_encrypted_proto(struct ndpi_detection_module_struct *ndpi_str, /* ******************************************* */ -void ndpi_set_tls_cert_expire_days(struct ndpi_detection_module_struct *ndpi_str, - u_int8_t num_days) { - if(ndpi_str) - ndpi_str->tls_certificate_expire_in_x_days = num_days; -} - -/* ******************************************* */ - u_int32_t ndpi_get_flow_error_code(struct ndpi_flow_struct *flow) { switch(flow->detected_protocol_stack[0] /* app_protocol */) { case NDPI_PROTOCOL_DNS: diff --git a/src/lib/protocols/ftp_control.c b/src/lib/protocols/ftp_control.c index 40378a4c6640..7f0ca42b1294 100644 --- a/src/lib/protocols/ftp_control.c +++ b/src/lib/protocols/ftp_control.c @@ -640,7 +640,7 @@ static void ndpi_check_ftp_control(struct ndpi_detection_module_struct *ndpi_str flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) { flow->ftp_control_stage = 0; } else if (flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 1 && - ndpi_struct->opportunistic_tls_ftp_enabled) { + ndpi_struct->cfg.smtp_opportunistic_tls_enabled) { flow->host_server_name[0] = '\0'; /* Remove any data set by other dissectors (eg. SMTP) */ /* Switch classification to FTPS */ ndpi_set_detected_protocol(ndpi_struct, flow, diff --git a/src/lib/protocols/mail_imap.c b/src/lib/protocols/mail_imap.c index a9eeaf64f86c..1b8b31287f00 100644 --- a/src/lib/protocols/mail_imap.c +++ b/src/lib/protocols/mail_imap.c @@ -110,7 +110,7 @@ static void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_ if(flow->l4.tcp.mail_imap_starttls == 1) { NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n"); ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS); - if(ndpi_struct->opportunistic_tls_imap_enabled) { + if(ndpi_struct->cfg.imap_opportunistic_tls_enabled) { NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n", flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]); /* We are done (in IMAP dissector): delegating TLS... */ diff --git a/src/lib/protocols/mail_pop.c b/src/lib/protocols/mail_pop.c index 045ff1c7c704..5741eba0e4f4 100644 --- a/src/lib/protocols/mail_pop.c +++ b/src/lib/protocols/mail_pop.c @@ -174,7 +174,7 @@ static void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct if(packet->payload[0] == '+' && flow->l4.tcp.mail_imap_starttls == 1) { NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n"); ndpi_int_mail_pop_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POPS); - if(ndpi_struct->opportunistic_tls_pop_enabled) { + if(ndpi_struct->cfg.pop_opportunistic_tls_enabled) { NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n", flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]); /* We are done (in POP dissector): delegating TLS... */ diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c index 980ebb8c9828..cf8e311461ac 100644 --- a/src/lib/protocols/mail_smtp.c +++ b/src/lib/protocols/mail_smtp.c @@ -418,7 +418,7 @@ int ndpi_extra_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_st 454 TLS not available due to temporary reason" */ - if(ndpi_struct->opportunistic_tls_smtp_enabled && + if(ndpi_struct->cfg.smtp_opportunistic_tls_enabled && packet->payload_packet_len > 3 && memcmp(packet->payload, "220", 3) == 0) { rc = 1; /* Switch classification to SMTPS, keeping the hostname sub-classification (if any) */ diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c index b3d83da8cdec..f3e55dfbca51 100644 --- a/src/lib/protocols/stun.c +++ b/src/lib/protocols/stun.c @@ -403,7 +403,7 @@ static int stun_search_again(struct ndpi_detection_module_struct *ndpi_struct, } else if(first_byte <= 63) { NDPI_LOG_DBG(ndpi_struct, "DTLS\n"); - if(ndpi_struct->opportunistic_tls_stun_enabled && + if(ndpi_struct->cfg.stun_opportunistic_tls_enabled && is_dtls(packet->payload, packet->payload_packet_len, &unused)) { /* Process this DTLS packet via TLS/DTLS code but keep using STUN dissection. diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index 613303398ff7..98e34243aa04 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -536,7 +536,7 @@ void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct snprintf(str, sizeof(str), "%s - %s", b, e); ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_CERTIFICATE_EXPIRED, str); /* Certificate expired */ } else if((time_sec > flow->protos.tls_quic.notBefore) - && (time_sec > (flow->protos.tls_quic.notAfter - (ndpi_struct->tls_certificate_expire_in_x_days * 86400)))) { + && (time_sec > (flow->protos.tls_quic.notAfter - (ndpi_struct->cfg.tls_certificate_expire_in_x_days * 86400)))) { char str[96], b[32], e[32]; struct tm result; time_t theTime; @@ -1154,7 +1154,7 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct, printf("*** [TLS Block] No more blocks\n"); #endif /* An ookla flow? */ - if((ndpi_struct->aggressiveness_ookla & NDPI_AGGRESSIVENESS_OOKLA_TLS) && /* Feature enabled */ + if((ndpi_struct->cfg.ookla_aggressiveness & AGGRESSIVENESS_OOKLA_TLS) && /* Feature enabled */ (!something_went_wrong && flow->tls_quic.certificate_processed == 1 && flow->protos.tls_quic.hello_processed == 1) && /* TLS handshake found without errors */ diff --git a/tests/cfgs/disable_aggressiveness/config.txt b/tests/cfgs/disable_aggressiveness/config.txt index 1504f51be82b..6a4f2409946d 100644 --- a/tests/cfgs/disable_aggressiveness/config.txt +++ b/tests/cfgs/disable_aggressiveness/config.txt @@ -1 +1 @@ --Z ookla:0 +--cfg=ookla,aggressiveness,0x0