From def08c3c848be5aceb309e20b13bfaa25d984818 Mon Sep 17 00:00:00 2001 From: Nardi Ivan Date: Fri, 1 Dec 2023 11:50:25 +0100 Subject: [PATCH] Add generic functions for library configuration This is a new way to provide (more) configuration options in nDPI. The idea is to have a simple way to configure (most of) nDPI: only one function (`ndpi_set_config()`) to set any configuration parameters (in the present or on in the future) and this function prototype is as agnostic as possible. This way, anytime we need to add a new configuration parameter: * we don't need to add two public functions (a getter and a setter) * we don't break API/ABI compatibility of the library; even changing the parameter type (from integer to a list of integer, for example) doesn't break the compatibility. Two examples of how to extend the configuration are provided: * the ability to enable/disable the extraction of the sha1 fingerprint of the TLS certificates. * the upper limit on the number of packets per flow that will be subject to inspection --- Makefile.am | 2 +- doc/configuration_parameters.md | 9 + example/ndpiReader.c | 98 ++++++++- fuzz/fuzz_config.cpp | 7 +- src/include/ndpi_api.h | 9 + src/include/ndpi_define.h.in | 1 - src/include/ndpi_typedefs.h | 9 +- src/lib/ndpi_main.c | 202 +++++++++++++++++- src/lib/ndpi_private.h | 3 +- src/lib/protocols/tls.c | 60 +++--- tests/cfgs/disable_metadata/config.txt | 1 + .../pcap/tls_verylong_certificate.pcap | 1 + .../result/tls_verylong_certificate.pcap.out | 33 +++ tests/cfgs/packets_limit_per_flow/config.txt | 1 + .../pcap/tls_verylong_certificate.pcap | 1 + .../result/tls_verylong_certificate.pcap.out | 33 +++ 16 files changed, 415 insertions(+), 55 deletions(-) create mode 100644 doc/configuration_parameters.md create mode 100644 tests/cfgs/disable_metadata/config.txt create mode 120000 tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap create mode 100644 tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out create mode 100644 tests/cfgs/packets_limit_per_flow/config.txt create mode 120000 tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap create mode 100644 tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out diff --git a/Makefile.am b/Makefile.am index cf6e13d4745a..871c761bd3d5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,7 +14,7 @@ EXTRA_DIST = README.md README.fuzzer.md CHANGELOG.md CONTRIBUTING.md \ rrdtool/Makefile.in rrdtool/README.txt rrdtool/rrd_anomaly.c rrdtool/rrd_similarity.c \ doc/requirements.txt doc/conf.py doc/flow_risks.rst doc/protocols.rst doc/guide/nDPI_QuickStartGuide.pages \ doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst \ - doc/Makefile doc/what_is_ndpi.rst doc/FAQ.md \ + doc/Makefile doc/what_is_ndpi.rst doc/FAQ.md doc/configuration_parameters.md \ python/DEV_GUIDE.md python/dev_requirements.txt python/ndpi_example.py python/ndpi/__init__.py \ python/ndpi/ndpi_build.py python/ndpi/ndpi.py python/README.md \ python/requirements.txt python/setup.py python/tests.py \ diff --git a/doc/configuration_parameters.md b/doc/configuration_parameters.md new file mode 100644 index 000000000000..70a4028199d1 --- /dev/null +++ b/doc/configuration_parameters.md @@ -0,0 +1,9 @@ + +# Configuration knobs + +TODO + +| Protocol | Parameter | Default value | Min value | Max value | Description | +| ------ | ------ | ------ | ------ | ------ | ------ | +| NULL | "packets_limit_per_flow" | 32 | 0 | 255 | The upper limit on the number of packets per flow that will be subject to DPI, after which classification will be considered complete (0 = no limit)| +| "tls" | "metadata.sha1_fingerprint.enable" | 1 | NULL | NULL | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked | diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 0b762e23c570..eeb48d19f78d 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -96,6 +96,16 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0, num_bin_cluster u_int8_t verbose = 0, enable_flow_stats = 0; int stun_monitoring_pkts_to_process = -1; /* Default */ int stun_monitoring_flags = -1; /* Default */ + +struct cfg { + char *proto; + char *param; + char *value; +}; +#define MAX_NUM_CFGS 16 +static struct cfg cfgs[MAX_NUM_CFGS]; +static int num_cfgs = 0; + int nDPI_LogLevel = 0; char *_debug_protocols = NULL; char *_disabled_protocols = NULL; @@ -594,11 +604,20 @@ static void help(u_int long_help) { " --lru-cache-ttl=NAME:size | Specify the TTL [in seconds] for this LRU cache (0 to disable it). This flag can be used multiple times\n" " --stun-monitoring=: | Configure STUN monitoring: keep monitoring STUN session for more pkts looking for RTP\n" " | (0:0 to disable the feature); set the specified features in \n" + " --cfg=proto,param,value | Configure the specific attribute of this protocol\n" , human_readeable_string_len, min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection, max_num_reported_top_payloads, max_num_tcp_dissected_pkts, max_num_udp_dissected_pkts); + NDPI_PROTOCOL_BITMASK all; + ndpi_info_mod = ndpi_init_detection_module(init_prefs); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all); + + printf("\nProtocols configuration parameters:\n"); + ndpi_dump_config(ndpi_info_mod, stdout); + printf("\nLRU Cache names: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom\n"); #ifndef WIN32 @@ -621,24 +640,18 @@ static void help(u_int long_help) { ndpi_detection_get_sizeof_ndpi_flow_struct(), sizeof(((struct ndpi_flow_struct *)0)->protos)); - NDPI_PROTOCOL_BITMASK all; - - ndpi_info_mod = ndpi_init_detection_module(init_prefs); printf("\n\nnDPI supported protocols:\n"); printf("%3s %-22s %-10s %-8s %-12s %s\n", "Id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category"); num_threads = 1; - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all); - ndpi_dump_protocols(ndpi_info_mod, stdout); printf("\n\nnDPI supported risks:\n"); ndpi_dump_risks_score(stdout); - - ndpi_exit_detection_module(ndpi_info_mod); } + + ndpi_exit_detection_module(ndpi_info_mod); exit(!long_help); } @@ -649,6 +662,8 @@ static void help(u_int long_help) { #define OPTLONG_VALUE_STUN_MONITORING 2000 +#define OPTLONG_VALUE_CFG 3000 + static struct option longopts[] = { /* mandatory extcap options */ { "extcap-interfaces", no_argument, NULL, '0'}, @@ -694,6 +709,8 @@ static struct option longopts[] = { { "lru-cache-ttl", required_argument, NULL, OPTLONG_VALUE_LRU_CACHE_TTL}, { "stun-monitoring", required_argument, NULL, OPTLONG_VALUE_STUN_MONITORING}, + { "cfg", required_argument, NULL, OPTLONG_VALUE_CFG}, + {0, 0, 0, 0} }; @@ -950,6 +967,42 @@ static int parse_two_unsigned_integer(char *param, u_int32_t *num1, u_int32_t *n return -1; } +static int parse_three_strings(char *param, char **s1, char **s2, char **s3) +{ + char *saveptr, *tmp_str, *s1_str, *s2_str = NULL, *s3_str; + + tmp_str = ndpi_strdup(param); + if(tmp_str) { + if(param[0] == ',') { /* First parameter might be missing */ + s1_str = NULL; + s2_str = strtok_r(tmp_str, ",", &saveptr); + } else { + s1_str = strtok_r(tmp_str, ",", &saveptr); + if(s1_str) { + s2_str = strtok_r(NULL, ",", &saveptr); + } + } + if(s2_str) { + s3_str = strtok_r(NULL, ",", &saveptr); + if(s3_str) { + *s1 = ndpi_strdup(s1_str); + *s2 = ndpi_strdup(s2_str); + *s3 = ndpi_strdup(s3_str); + ndpi_free(tmp_str); + if(!s1 || !s2 || !s3) { + ndpi_free(s1); + ndpi_free(s2); + ndpi_free(s3); + return -1; + } + return 0; + } + } + } + ndpi_free(tmp_str); + return -1; +} + /* ********************************** */ /** @@ -968,6 +1021,7 @@ static void parseOptions(int argc, char **argv) { #endif int cache_idx, cache_size, cache_ttl; u_int32_t num_pkts, flags; + char *s1, *s2, *s3; #ifdef USE_DPDK { @@ -1316,6 +1370,18 @@ static void parseOptions(int argc, char **argv) { stun_monitoring_flags = flags; break; + case OPTLONG_VALUE_CFG: + if(num_cfgs >= MAX_NUM_CFGS || + parse_three_strings(optarg, &s1, &s2, &s3) == -1) { + printf("Invalid parameter [%s] [num:%d/%d]\n", optarg, num_cfgs, MAX_NUM_CFGS); + exit(1); + } + cfgs[num_cfgs].proto = s1; + cfgs[num_cfgs].param = s2; + cfgs[num_cfgs].value = s3; + num_cfgs++; + break; + default: #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### Unknown option -%c: skipping it #### \n", opt); @@ -2669,7 +2735,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct, static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { NDPI_PROTOCOL_BITMASK enabled_bitmask; struct ndpi_workflow_prefs prefs; - int i; + int i, rc; memset(&prefs, 0, sizeof(prefs)); prefs.decode_tunnels = decode_tunnels; @@ -2757,6 +2823,14 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]); } + for(i = 0; i < num_cfgs; i++) { + rc = ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, + cfgs[i].proto, cfgs[i].param, cfgs[i].value); + if (rc != 0) + fprintf(stderr, "Error setting config [%s][%s][%s]: %d\n", + cfgs[i].proto, cfgs[i].param, cfgs[i].value, rc); + } + if(stun_monitoring_pkts_to_process != -1 && stun_monitoring_flags != -1) ndpi_set_monitoring_state(ndpi_thread_info[thread_id].workflow->ndpi_struct, NDPI_PROTOCOL_STUN, @@ -5684,6 +5758,12 @@ int main(int argc, char **argv) { ndpi_free(_debug_protocols); ndpi_free(_disabled_protocols); + for(i = 0; i < num_cfgs; i++) { + ndpi_free(cfgs[i].proto); + ndpi_free(cfgs[i].param); + ndpi_free(cfgs[i].value); + } + #ifdef DEBUG_TRACE if(trace) fclose(trace); #endif diff --git a/fuzz/fuzz_config.cpp b/fuzz/fuzz_config.cpp index 0abf63515c40..eea0ec0677cf 100644 --- a/fuzz/fuzz_config.cpp +++ b/fuzz/fuzz_config.cpp @@ -35,7 +35,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { 1 + /* TLS cert expire */ 6 + /* files */ ((NDPI_LRUCACHE_MAX + 1) * 5) + /* LRU caches */ - 2 + 1 + 4 + /* ndpi_set_detection_preferences() */ + 2 + 1 + /* ndpi_set_detection_preferences() */ 1 + 3 + 1 + 3 + /* Monitoring */ 7 + /* Opportunistic tls */ 2 + /* Pid */ @@ -67,6 +67,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { ndpi_set_user_data(ndpi_info_mod, (void *)0xabcdabcd); /* Twice to trigger overwriting */ ndpi_get_user_data(ndpi_info_mod); + /* TODO: ndpi_config_set */ + ndpi_set_tls_cert_expire_days(ndpi_info_mod, fuzzed_data.ConsumeIntegral()); if(fuzzed_data.ConsumeBool()) @@ -102,9 +104,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { if(fuzzed_data.ConsumeBool()) ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_enable_tls_block_dissection, 0 /* unused */); - if(fuzzed_data.ConsumeBool()) - ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_max_packets_to_process, - fuzzed_data.ConsumeIntegralInRange(0, (1 << 16))); ndpi_set_detection_preferences(ndpi_info_mod, static_cast(0xFF), 0xFF); /* Invalid preference */ diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 0eef7bae2a20..5d70bed5bbf0 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -2154,6 +2154,15 @@ extern "C" { /* ******************************* */ + int ndpi_set_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, const char *value); + char *ndpi_get_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, char *buf, int buf_len); + char *ndpi_dump_config(struct ndpi_detection_module_struct *ndpi_str, + FILE *fd); + + /* ******************************* */ + /* Can't call libc functions from kernel space, define some stub instead */ #define ndpi_isalpha(ch) (((ch) >= 'a' && (ch) <= 'z') || ((ch) >= 'A' && (ch) <= 'Z')) diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in index 4d61d7130e40..c0a0f5500332 100644 --- a/src/include/ndpi_define.h.in +++ b/src/include/ndpi_define.h.in @@ -153,7 +153,6 @@ /* misc definitions */ #define NDPI_DEFAULT_MAX_TCP_RETRANSMISSION_WINDOW_SIZE 0x10000 -#define NDPI_DEFAULT_MAX_NUM_PKTS_PER_FLOW_TO_DISSECT 32 /* TODO: rebuild all memory areas to have a more aligned memory block here */ diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 3b29cb046025..319f392cdac2 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1068,7 +1068,6 @@ typedef enum { typedef enum { ndpi_pref_direction_detect_disable = 0, - ndpi_pref_max_packets_to_process, ndpi_pref_enable_tls_block_dissection, /* nDPI considers only those blocks past the certificate exchange */ } ndpi_detection_preference; @@ -1154,6 +1153,14 @@ struct ndpi_risk_information { char *info; }; +struct ndpi_detection_module_config_struct { + int max_packets_to_process; + + /* TLS */ + int sha1_fingerprint_enabled; + +}; + struct ndpi_flow_struct { u_int16_t detected_protocol_stack[NDPI_PROTOCOL_SIZE]; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 2dbf92e64d9b..cde495bac26b 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -24,6 +24,7 @@ #include #include #include +#include #ifdef __APPLE__ #include @@ -223,6 +224,8 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str); static void ndpi_enabled_callbacks_init(struct ndpi_detection_module_struct *ndpi_str, const NDPI_PROTOCOL_BITMASK *dbm, int count_only); +static void set_default_config(struct ndpi_detection_module_config_struct *cfg); + /* ****************************************** */ ndpi_custom_dga_predict_fctn ndpi_dga_function = NULL; @@ -974,13 +977,6 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_str ndpi_str->skip_tls_blocks_until_change_cipher = 1; break; - case ndpi_pref_max_packets_to_process: - if(value > 0xFFFF) { - return(-1); - } - ndpi_str->max_packets_to_process = value; - break; - default: return(-1); } @@ -3241,7 +3237,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs } } - ndpi_str->max_packets_to_process = NDPI_DEFAULT_MAX_NUM_PKTS_PER_FLOW_TO_DISSECT; + set_default_config(&ndpi_str->cfg); NDPI_BITMASK_SET_ALL(ndpi_str->detection_bitmask); ndpi_str->user_data = NULL; @@ -7745,7 +7741,7 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio return(ret); } - if(ndpi_str->max_packets_to_process > 0 && flow->num_processed_pkts >= ndpi_str->max_packets_to_process) { + if(ndpi_str->cfg.max_packets_to_process > 0 && flow->num_processed_pkts >= ndpi_str->cfg.max_packets_to_process) { flow->extra_packets_func = NULL; /* To allow ndpi_extra_dissection_possible() to fail */ flow->fail_with_unknown = 1; return(ret); /* Avoid spending too much time with this flow */ @@ -10562,3 +10558,191 @@ void *ndpi_get_user_data(struct ndpi_detection_module_struct *ndpi_str) return ndpi_str->user_data; return NULL; } + +/* ******************************************************************** */ + +static int _set_param_enable_disable(void *_variable, const char *value) +{ + int *variable = (int *)_variable; + + if(strcmp(value, "1") == 0 || + strcmp(value, "enable") == 0) { + *variable = 1; + return 0; + } + if(strcmp(value, "0") == 0 || + strcmp(value, "disable") == 0) { + *variable = 0; + return 0; + } + return -1; +} + +static int _set_param_int(void *_variable, const char *value, const char *min_value, const char *max_value) +{ + int *variable = (int *)_variable; + char *endptr; + long val; + + errno = 0; /* To distinguish success/failure after call */ + val = strtol(value, &endptr, 10); + + /* Check for various possible errors */ + if((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || + (errno != 0 && val == 0)) { + return -1; + } + if (endptr == value) { + /* No digits were found */ + return -1; + } + /* If we got here, strtol() successfully parsed a number */ + *variable = val; + + /* Min and max values are set in the code, so we can convert them + to integers without too many checks...*/ + if(min_value && max_value && + (val < atoi(min_value) || val > atoi(max_value))) + return -1; + + return 0; +} + +/* It can be used for CFG_PARAM_ENABLE_DISABLE parameters, too */ +static char *_get_param_int(void *_variable, char *buf, int buf_len) +{ + int *variable = (int *)_variable; + + snprintf(buf, buf_len, "%d", *variable); + buf[buf_len - 1] = '\0'; + return buf; +} + + +typedef int (*cfg_fn)(void *variable, const char *value); + +enum cfg_param_type { + CFG_PARAM_ENABLE_DISABLE = 0, + CFG_PARAM_INT = 1, +}; + +#define __OFF(a) offsetof(struct ndpi_detection_module_config_struct, a) + +static const struct cfg_param { + char *proto; + char *param; + char *default_value; + char *min_value; + char *max_value; + enum cfg_param_type type; + int offset; +} cfg_params[] = { + /* Per-protocol parameters */ + + /* An example of metadata configuration (yes/no) */ + { "tls", "metadata.sha1_fingerprint.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(sha1_fingerprint_enabled) }, + + /* Global parameters */ + + /* An example of integer configuration */ + { NULL, "packets_limit_per_flow", "32", "0", "255", CFG_PARAM_INT, __OFF(max_packets_to_process) }, + + { NULL, NULL, NULL, NULL, NULL, 0, -1 }, +}; + +#undef __OFF + +static void set_default_config(struct ndpi_detection_module_config_struct *cfg) +{ + const struct cfg_param *c; + + for(c = &cfg_params[0]; c && c->param; c++) { + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + _set_param_enable_disable((void *)((char *)cfg + c->offset), c->default_value); + break; + case CFG_PARAM_INT: + _set_param_int((void *)((char *)cfg + c->offset), c->default_value, c->min_value, c->max_value); + break; + } + } +} + +int ndpi_set_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, const char *value) +{ + const struct cfg_param *c; + + if(!ndpi_str || !param || !value) + return -2; + + NDPI_LOG_DBG(ndpi_str, "Set [%s][%s][%s]\n", proto, param, value); + + for(c = &cfg_params[0]; c && c->param; c++) { + if(((proto == NULL && c->proto == NULL) || + (proto && c->proto && strcmp(proto, c->proto) == 0)) && + strcmp(param, c->param) == 0) { + + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + return _set_param_enable_disable((void *)((char *)&ndpi_str->cfg + c->offset), value); + case CFG_PARAM_INT: + return _set_param_int((void *)((char *)&ndpi_str->cfg + c->offset), value, c->min_value, c->max_value); + } + } + } + return -3; +} + +char *ndpi_get_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, char *buf, int buf_len) +{ + const struct cfg_param *c; + + if(!ndpi_str || !param || !buf || buf_len <= 0) + return NULL; + + NDPI_LOG_ERR(ndpi_str, "Get [%s][%s]\n", proto, param); + + for(c = &cfg_params[0]; c && c->param; c++) { + if(((proto == NULL && c->proto == NULL) || + (proto && c->proto && strcmp(proto, c->proto) == 0)) && + strcmp(param, c->param) == 0) { + + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + case CFG_PARAM_INT: + return _get_param_int((void *)((char *)&ndpi_str->cfg + c->offset), buf, buf_len); + } + } + } + return NULL; +} + +char *ndpi_dump_config(struct ndpi_detection_module_struct *ndpi_str, + FILE *fd) +{ + const struct cfg_param *c; + char buf[64]; + + if(!ndpi_str || !fd) + return NULL; + + fprintf(fd, " Protocol (empty/NULL for global knobs), parameter, value, [default value], [min value, max_value]\n"); + + for(c = &cfg_params[0]; c && c->param; c++) { + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + case CFG_PARAM_INT: + fprintf(fd, " *) %s %s: %s [%s]", + c->proto ? c->proto : "NULL", + c->param, + _get_param_int((void *)((char *)&ndpi_str->cfg + c->offset), buf, sizeof(buf)), + c->default_value); + if(c->min_value && c->max_value) + fprintf(fd, " [%s-%s]", c->min_value, c->max_value); + fprintf(fd, "\n"); + } + } + return NULL; +} diff --git a/src/lib/ndpi_private.h b/src/lib/ndpi_private.h index 8a06804e2ad4..938bef5e6891 100644 --- a/src/lib/ndpi_private.h +++ b/src/lib/ndpi_private.h @@ -126,7 +126,6 @@ struct ndpi_detection_module_struct { NDPI_PROTOCOL_BITMASK detection_bitmask; u_int64_t current_ts; - u_int16_t max_packets_to_process; u_int16_t num_tls_blocks_to_follow; u_int8_t skip_tls_blocks_until_change_cipher:1, enable_ja3_plus:1, _notused:6; u_int8_t tls_certificate_expire_in_x_days; @@ -204,6 +203,8 @@ struct ndpi_detection_module_struct { u_int8_t ip_version_limit; + struct ndpi_detection_module_config_struct cfg; + /* NDPI_PROTOCOL_TINC */ struct cache *tinc_cache; diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index b86d51f54b83..1200a7646351 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -763,7 +763,6 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t certificates_length, length = (packet->payload[1] << 16) + (packet->payload[2] << 8) + packet->payload[3]; u_int32_t certificates_offset = 7 + (is_dtls ? 8 : 0); u_int8_t num_certificates_found = 0; - SHA1_CTX srv_cert_fingerprint_ctx ; #ifdef DEBUG_TLS printf("[TLS] %s() [payload_packet_len=%u][direction: %u][%02X %02X %02X %02X %02X %02X...]\n", @@ -815,9 +814,6 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, #endif if(num_certificates_found++ == 0) /* Dissect only the first certificate that is the one we care */ { - /* For SHA-1 we take into account only the first certificate and not all of them */ - - SHA1Init(&srv_cert_fingerprint_ctx); #ifdef DEBUG_CERTIFICATE_HASH { @@ -830,36 +826,42 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, } #endif - SHA1Update(&srv_cert_fingerprint_ctx, - &packet->payload[certificates_offset], - certificate_len); - - SHA1Final(flow->protos.tls_quic.sha1_certificate_fingerprint, &srv_cert_fingerprint_ctx); - - flow->protos.tls_quic.fingerprint_set = 1; - - uint8_t * sha1 = flow->protos.tls_quic.sha1_certificate_fingerprint; - const size_t sha1_siz = sizeof(flow->protos.tls_quic.sha1_certificate_fingerprint); - char sha1_str[20 /* sha1_siz */ * 2 + 1]; - static const char hexalnum[] = "0123456789ABCDEF"; - size_t i; - for (i = 0; i < sha1_siz; ++i) { - u_int8_t lower = (sha1[i] & 0x0F); - u_int8_t upper = (sha1[i] & 0xF0) >> 4; - sha1_str[i*2] = hexalnum[upper]; - sha1_str[i*2 + 1] = hexalnum[lower]; - } - sha1_str[sha1_siz * 2] = '\0'; + /* For SHA-1 we take into account only the first certificate and not all of them */ + if(ndpi_struct->cfg.sha1_fingerprint_enabled) { + SHA1_CTX srv_cert_fingerprint_ctx ; + + SHA1Init(&srv_cert_fingerprint_ctx); + SHA1Update(&srv_cert_fingerprint_ctx, + &packet->payload[certificates_offset], + certificate_len); + + SHA1Final(flow->protos.tls_quic.sha1_certificate_fingerprint, &srv_cert_fingerprint_ctx); + + flow->protos.tls_quic.fingerprint_set = 1; + + uint8_t * sha1 = flow->protos.tls_quic.sha1_certificate_fingerprint; + const size_t sha1_siz = sizeof(flow->protos.tls_quic.sha1_certificate_fingerprint); + char sha1_str[20 /* sha1_siz */ * 2 + 1]; + static const char hexalnum[] = "0123456789ABCDEF"; + size_t i; + for (i = 0; i < sha1_siz; ++i) { + u_int8_t lower = (sha1[i] & 0x0F); + u_int8_t upper = (sha1[i] & 0xF0) >> 4; + sha1_str[i*2] = hexalnum[upper]; + sha1_str[i*2 + 1] = hexalnum[lower]; + } + sha1_str[sha1_siz * 2] = '\0'; #ifdef DEBUG_TLS - printf("[TLS] SHA-1: %s\n", sha1_str); + printf("[TLS] SHA-1: %s\n", sha1_str); #endif - if(ndpi_struct->malicious_sha1_hashmap != NULL) { - u_int16_t rc1 = ndpi_hash_find_entry(ndpi_struct->malicious_sha1_hashmap, sha1_str, sha1_siz * 2, NULL); + if(ndpi_struct->malicious_sha1_hashmap != NULL) { + u_int16_t rc1 = ndpi_hash_find_entry(ndpi_struct->malicious_sha1_hashmap, sha1_str, sha1_siz * 2, NULL); - if(rc1 == 0) - ndpi_set_risk(ndpi_struct, flow, NDPI_MALICIOUS_SHA1_CERTIFICATE, sha1_str); + if(rc1 == 0) + ndpi_set_risk(ndpi_struct, flow, NDPI_MALICIOUS_SHA1_CERTIFICATE, sha1_str); + } } processCertificateElements(ndpi_struct, flow, certificates_offset, certificate_len); diff --git a/tests/cfgs/disable_metadata/config.txt b/tests/cfgs/disable_metadata/config.txt new file mode 100644 index 000000000000..1e5e30bf8387 --- /dev/null +++ b/tests/cfgs/disable_metadata/config.txt @@ -0,0 +1 @@ +--cfg=tls,metadata.sha1_fingerprint.enable,0 diff --git a/tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap b/tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap new file mode 120000 index 000000000000..2f722f28ed61 --- /dev/null +++ b/tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap @@ -0,0 +1 @@ +../../default/pcap/tls_verylong_certificate.pcap \ No newline at end of file diff --git a/tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out b/tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out new file mode 100644 index 000000000000..ff294999ed2e --- /dev/null +++ b/tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out @@ -0,0 +1,33 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 11 (11.00 pkts/flow) +Confidence DPI : 1 (flows) +Num dissector calls: 1 (1.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 1/1 (search/found) +Automa domain: 1/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 1/1 (search/found) +Patricia risk mask: 0/0 (search/found) +Patricia risk mask IPv6: 0/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia risk IPv6: 0/0 (search/found) +Patricia protocols: 2/0 (search/found) +Patricia protocols IPv6: 0/0 (search/found) + +Cybersec 48 22229 1 + +JA3 Host Stats: + IP Address # JA3C + 1 192.168.1.160 1 + + + 1 TCP 192.168.1.160:54804 <-> 151.101.66.49:443 [proto: 91.283/TLS.Cybersec][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 11][cat: Cybersecurity/33][24 pkts/2404 bytes <-> 24 pkts/19825 bytes][Goodput ratio: 35/92][0.09 sec][Hostname/SNI: feodotracker.abuse.ch][(Advertised) ALPNs: http/1.1][(Negotiated) ALPN: http/1.1][bytes ratio: -0.784 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/4 15/21 5/7][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 100/826 583/1434 109/662][TLSv1.2][JA3C: 2a26b1a62e40d25d4de3babc9d532f30][ServerNames: p2.shared.global.fastly.net,*.12wbt.com,*.2bleacherreport.com,*.3bleacherreport.com,*.4bleacherreport.com,*.8bleacherreport.com,*.abuse.ch,*.acdn-it.ps-pantheon.com,*.cdn.livingmap.com,*.content.plastiq.com,*.dimensions.ai,*.dollarshaveclub.co.uk,*.dollarshaveclub.com,*.dontpayfull.com,*.ebisubook.com,*.foreignaffairs.com,*.fs.jibjab.com,*.fs.unitprints.com,*.ggleap.com,*.goodeggs.com,*.huevosbuenos.com,*.indy.myomnigon.com,*.jwatch.org,*.kingsfordcharcoal.com.au,*.lancenters.com,*.madebywe.com,*.minirodini.com,*.modcloth.net,*.orionlabs.io,*.ps-pantheon.com,*.scodle.com,*.steelseries.com,*.theforeman.org,*.uploads.eversign.com,*.uploads.schoox.com,*.vts.com,*.x.stg1.ebisubook.com,*.yang2020.com,12wbt.com,2bleacherreport.com,3bleacherreport.com,4bleacherreport.com,8bleacherreport.com,abuse.ch,brita.com,cdn.fwupd.org,cdn.livingmap.com,cdn.seated.com,cdn.skillacademy.com,clinicaloptions.com,clorox.com,content-preprod.beaverbrooksweb2.co.uk,content.beaverbrooks.co.uk,content.plastiq.com,coolmathgames.com,copterroyale.coolmathgames.com,d8-dev.coolmathgames.com,deflyio.coolmathgames.com,delivery-api.evadacms.com,dimensions.ai,dollarshaveclub.co.uk,dollarshaveclub.com,dontpayfull.com,eluniverso.com,email.amg-group.co,email.tekoforlife.co.uk,feedmarket.fr,freshstep.com,ggleap.com,goodeggs.com,heap.io,huevosbuenos.com,identity.linuxfoundation.org,joebiden.com,jwatch.org,kingsford.co.nz,kingsfordcharcoal.com.au,lancenters.com,lists.linuxfoundation.org,m-stage.coolmathgames.com,m.coolmathgames.com,madebywe.com,minirodini.com,modcloth.net,orionlabs.io,puritanmedproducts.com,reviews.org,rg-video-staging.ruangguru.com,rg-video.ruangguru.com,ruangguru.com,scodle.com,stage.coolmathgames.com,staging.appblade.com,steelseries.com,stg.platform.eluniverso.com,test.brita.com,test.heap.io,test.joebiden.com,test.ruangguru.com,theforeman.org,video-cdn.quipper.com,videos.calcworkshop.com,vts.com,www.101network.com,www.autos101.com,www.brita.com,www.clorox.com,www.collider.com,www.coolmathgames.com,www.eluniverso.com,www.flinto.com,www.freshstep.com,www.heap.io,www.holagente.com,www.icsydney.com.au,www.joebiden.com,www.kingsford.co.nz,www.mrnatty.com,www.myjewellerystory.com.au,www.myjs.com,www.netacea.com,www.parenting101.com,www.puritanmedproducts.com,www.reviews.org,www.sba.sa,www.shashatcom.sa,www.uat.ontariocolleges.ca,www.vacation101.com,www.walterspeople.co.uk,www.westwayelectricsupply.com][JA3S: ae53107a2e47ea20c72ac44821a728bf][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign CloudSSL CA - SHA256 - G3][Subject: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=p2.shared.global.fastly.net][Firefox][Validity: 2019-11-19 01:31:22 - 2020-08-29 17:19:32][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 12,16,0,4,0,4,4,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0,0,0,0] diff --git a/tests/cfgs/packets_limit_per_flow/config.txt b/tests/cfgs/packets_limit_per_flow/config.txt new file mode 100644 index 000000000000..0e73f76c571a --- /dev/null +++ b/tests/cfgs/packets_limit_per_flow/config.txt @@ -0,0 +1 @@ +--cfg=,packets_limit_per_flow,4 diff --git a/tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap b/tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap new file mode 120000 index 000000000000..2f722f28ed61 --- /dev/null +++ b/tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap @@ -0,0 +1 @@ +../../default/pcap/tls_verylong_certificate.pcap \ No newline at end of file diff --git a/tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out b/tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out new file mode 100644 index 000000000000..b7120d6dc291 --- /dev/null +++ b/tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out @@ -0,0 +1,33 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 5 (5.00 pkts/flow) +Confidence DPI : 1 (flows) +Num dissector calls: 1 (1.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 1/1 (search/found) +Automa domain: 1/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 1/1 (search/found) +Patricia risk mask: 0/0 (search/found) +Patricia risk mask IPv6: 0/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia risk IPv6: 0/0 (search/found) +Patricia protocols: 2/0 (search/found) +Patricia protocols IPv6: 0/0 (search/found) + +Cybersec 48 22229 1 + +JA3 Host Stats: + IP Address # JA3C + 1 192.168.1.160 1 + + + 1 TCP 192.168.1.160:54804 <-> 151.101.66.49:443 [proto: 91.283/TLS.Cybersec][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 5][cat: Cybersecurity/33][24 pkts/2404 bytes <-> 24 pkts/19825 bytes][Goodput ratio: 35/92][0.09 sec][Hostname/SNI: feodotracker.abuse.ch][(Advertised) ALPNs: http/1.1][bytes ratio: -0.784 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/4 15/21 5/7][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 100/826 583/1434 109/662][TLSv1.2][JA3C: 2a26b1a62e40d25d4de3babc9d532f30][Firefox][Plen Bins: 12,16,0,4,0,4,4,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0,0,0,0]