Skip to content

Commit

Permalink
config: move LRU cache configurations to the new API
Browse files Browse the repository at this point in the history
  • Loading branch information
@IvanNardi
IvanNardi committed Jan 9, 2024
1 parent b9fca3e commit 5c13964
Show file tree
Hide file tree
Showing 8 changed files with 164 additions and 333 deletions.
2 changes: 2 additions & 0 deletions doc/configuration_parameters.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ TODO
| NULL | "packets_limit_per_flow" | 32 | 0 | 255 | The upper limit on the number of packets per flow that will be subject to DPI, after which classification will be considered complete (0 = no limit) |
| NULL | "filename.config" | NULL | NULL | NULL | Name of the file containing a list of configuration knobs itself (one per line)!. Useful to configure nDPI via text file instead of via API |
| "tls" | "metadata.sha1_fingerprint.enable" | 1 | NULL | NULL | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked |
| NULL | "lru.$CACHE_NAME.size" | See description | 0 | 16777215 | Set the size (in number of elements) of the specified LRU cache (0 = the cache is disabled). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom. The default value is "32768" for the bittorrent cache, "512" for the zoom cache and "1024" for all the other caches |
| NULL | "lru.$CACHE_NAME.ttl" | See description | 0 | 16777215 | Set the TTL (in seconds) for the elements of the specified LRU cache (0 = the elements never explicitly expire). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom. The default value is "120" for the ookla cache, "60" for the msteams and stun_zoom caches and "0" for all the other caches |
2 changes: 2 additions & 0 deletions example/config.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,6 @@
,packets_limit_per_flow,32
tls,metadata.sha1_fingerprint.enable,1

,lru.bittorrent.ttl,0


95 changes: 0 additions & 95 deletions example/ndpiReader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,6 @@ int enable_malloc_bins = 0;
int max_malloc_bins = 14;
int malloc_size_stats = 0;

static int lru_cache_sizes[NDPI_LRUCACHE_MAX];
static int lru_cache_ttls[NDPI_LRUCACHE_MAX];

struct flow_info {
struct ndpi_flow_info *flow;
u_int16_t thread_id;
Expand Down Expand Up @@ -607,16 +604,12 @@ static void help(u_int long_help) {
" -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
" -M | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
" -Z proto:value | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n"
" --lru-cache-size=NAME:size | Specify the size for this LRU cache (0 to disable it). This flag can be used multiple times\n"
" --lru-cache-ttl=NAME:size | Specify the TTL [in seconds] for this LRU cache (0 to disable it). This flag can be used multiple times\n"
" --cfg=proto,param,value | Configure the specific attribute of this protocol\n"
,
human_readeable_string_len,
min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
max_num_reported_top_payloads, max_num_tcp_dissected_pkts, max_num_udp_dissected_pkts);

printf("\nLRU Cache names: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom\n");

NDPI_PROTOCOL_BITMASK all;
struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module(init_prefs);
NDPI_BITMASK_SET_ALL(all);
Expand Down Expand Up @@ -663,9 +656,6 @@ static void help(u_int long_help) {
}


#define OPTLONG_VALUE_LRU_CACHE_SIZE 1000
#define OPTLONG_VALUE_LRU_CACHE_TTL 1001

#define OPTLONG_VALUE_CFG 3000

static struct option longopts[] = {
Expand Down Expand Up @@ -709,9 +699,6 @@ static struct option longopts[] = {
{ "result-path", required_argument, NULL, 'w'},
{ "quiet", no_argument, NULL, 'q'},

{ "lru-cache-size", required_argument, NULL, OPTLONG_VALUE_LRU_CACHE_SIZE},
{ "lru-cache-ttl", required_argument, NULL, OPTLONG_VALUE_LRU_CACHE_TTL},

{ "cfg", required_argument, NULL, OPTLONG_VALUE_CFG},

{0, 0, 0, 0}
Expand Down Expand Up @@ -903,52 +890,6 @@ void printCSVHeader() {
fprintf(csv_fp, "\n");
}

static int cache_idx_from_name(const char *name)
{
if(strcmp(name, "ookla") == 0)
return NDPI_LRUCACHE_OOKLA;
if(strcmp(name, "bittorrent") == 0)
return NDPI_LRUCACHE_BITTORRENT;
if(strcmp(name, "zoom") == 0)
return NDPI_LRUCACHE_ZOOM;
if(strcmp(name, "stun") == 0)
return NDPI_LRUCACHE_STUN;
if(strcmp(name, "tls_cert") == 0)
return NDPI_LRUCACHE_TLS_CERT;
if(strcmp(name, "mining") == 0)
return NDPI_LRUCACHE_MINING;
if(strcmp(name, "msteams") == 0)
return NDPI_LRUCACHE_MSTEAMS;
if(strcmp(name, "stun_zoom") == 0)
return NDPI_LRUCACHE_STUN_ZOOM;
return -1;
}

static int parse_cache_param(char *param, int *cache_idx, int *param_value)
{
char *saveptr, *tmp_str, *cache_str, *param_str;
int idx;

tmp_str = ndpi_strdup(param);
if(tmp_str) {
cache_str = strtok_r(tmp_str, ":", &saveptr);
if(cache_str) {
param_str = strtok_r(NULL, ":", &saveptr);
if(param_str) {
idx = cache_idx_from_name(cache_str);
if(idx >= 0) {
*cache_idx = idx;
*param_value = atoi(param_str);
ndpi_free(tmp_str);
return 0;
}
}
}
}
ndpi_free(tmp_str);
return -1;
}

static int parse_two_unsigned_integer(char *param, u_int32_t *num1, u_int32_t *num2)
{
char *saveptr, *tmp_str, *num1_str, *num2_str;
Expand Down Expand Up @@ -1035,7 +976,6 @@ static void parseOptions(int argc, char **argv) {
u_int num_cores = sysconf(_SC_NPROCESSORS_ONLN);
#endif
#endif
int cache_idx, cache_size, cache_ttl;
char *s1, *s2, *s3;

#ifdef USE_DPDK
Expand All @@ -1052,11 +992,6 @@ static void parseOptions(int argc, char **argv) {
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++)
aggressiveness[i] = -1; /* Use the default value */

for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
lru_cache_sizes[i] = -1; /* Use the default value */
lru_cache_ttls[i] = -1; /* Use the default value */
}

while((opt = getopt_long(argc, argv,
"a:Ab:B:e:Ec:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:Rs:tu:v:V:n:rp:x:X:w:Z:q0123:456:7:89:m:MT:U:",
longopts, &option_idx)) != EOF) {
Expand Down Expand Up @@ -1364,22 +1299,6 @@ static void parseOptions(int argc, char **argv) {
if(max_num_udp_dissected_pkts < 3) max_num_udp_dissected_pkts = 3;
break;

case OPTLONG_VALUE_LRU_CACHE_SIZE:
if(parse_cache_param(optarg, &cache_idx, &cache_size) == -1) {
printf("Invalid parameter [%s]\n", optarg);
exit(1);
}
lru_cache_sizes[cache_idx] = cache_size;
break;

case OPTLONG_VALUE_LRU_CACHE_TTL:
if(parse_cache_param(optarg, &cache_idx, &cache_ttl) == -1) {
printf("Invalid parameter [%s]\n", optarg);
exit(1);
}
lru_cache_ttls[cache_idx] = cache_ttl;
break;

case OPTLONG_VALUE_CFG:
if(parse_three_strings(optarg, &s1, &s2, &s3) == -1 ||
__add_cfg(s1, s2, s3, 0) == -1) {
Expand Down Expand Up @@ -2889,20 +2808,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
if(_protoFilePath != NULL)
ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath);

/* Enable/disable/configure LRU caches size here */
for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
if(lru_cache_sizes[i] != -1)
ndpi_set_lru_cache_size(ndpi_thread_info[thread_id].workflow->ndpi_struct,
i, lru_cache_sizes[i]);
}

/* Enable/disable LRU caches TTL here */
for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
if(lru_cache_ttls[i] != -1)
ndpi_set_lru_cache_ttl(ndpi_thread_info[thread_id].workflow->ndpi_struct,
i, lru_cache_ttls[i]);
}

/* Set aggressiveness here */
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
if(aggressiveness[i] != -1)
Expand Down
92 changes: 81 additions & 11 deletions fuzz/fuzz_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
struct ndpi_detection_module_struct *ndpi_info_mod;
struct ndpi_flow_struct flow;
u_int8_t protocol_was_guessed;
u_int32_t i, num;
u_int32_t i;
u_int16_t random_proto, bool_value;
int random_value;
NDPI_PROTOCOL_BITMASK enabled_bitmask;
Expand Down Expand Up @@ -73,16 +73,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
if(fuzzed_data.ConsumeBool())
ndpi_load_ipv4_ptree(ndpi_info_mod, "ipv4_addresses.txt", NDPI_PROTOCOL_TLS);

for(i = 0; i < NDPI_LRUCACHE_MAX + 1; i++) { /* + 1 to test invalid type */
ndpi_set_lru_cache_size(ndpi_info_mod, static_cast<lru_cache_type>(i),
fuzzed_data.ConsumeIntegralInRange(0, (1 << 16) - 1));
ndpi_get_lru_cache_size(ndpi_info_mod, static_cast<lru_cache_type>(i), &num);

ndpi_set_lru_cache_ttl(ndpi_info_mod, static_cast<lru_cache_type>(i),
fuzzed_data.ConsumeIntegralInRange(0, (1 << 24) - 1));
ndpi_get_lru_cache_ttl(ndpi_info_mod, static_cast<lru_cache_type>(i), &num);
}

/* TODO: stub for geo stuff */
ndpi_load_geoip(ndpi_info_mod, NULL, NULL);

Expand Down Expand Up @@ -124,6 +114,86 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "packets_limit_per_flow", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.ookla.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.ookla.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.bittorrent.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.bittorrent.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.zoom.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.zoom.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.stun.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.stun.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.tls_cert.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.tls_cert.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.mining.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.mining.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.msteams.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.msteams.ttl", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 / 2); /* max / 2 instead of max + 1 to avoid oom on oss-fuzzer */
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.stun_zoom.size", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 16777215 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, NULL, "lru.stun_zoom.ttl", cfg_value);
}

ndpi_finalize_initialization(ndpi_info_mod);

Expand Down
14 changes: 0 additions & 14 deletions src/include/ndpi_api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1074,20 +1074,6 @@ extern "C" {
lru_cache_type cache_type,
struct ndpi_lru_cache_stats *stats);

int ndpi_get_lru_cache_size(struct ndpi_detection_module_struct *ndpi_struct,
lru_cache_type cache_type,
u_int32_t *num_entries);
int ndpi_set_lru_cache_size(struct ndpi_detection_module_struct *ndpi_struct,
lru_cache_type cache_type,
u_int32_t num_entries);

int ndpi_set_lru_cache_ttl(struct ndpi_detection_module_struct *ndpi_struct,
lru_cache_type cache_type,
u_int32_t ttl);
int ndpi_get_lru_cache_ttl(struct ndpi_detection_module_struct *ndpi_struct,
lru_cache_type cache_type,
u_int32_t *ttl);

int ndpi_set_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto, int value);
int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
Expand Down
Loading

0 comments on commit 5c13964

Please sign in to comment.