From 91259a700551e9e9b723180d4095aad150732c32 Mon Sep 17 00:00:00 2001 From: idealeer Date: Fri, 2 Apr 2021 13:14:19 +0800 Subject: [PATCH] XMap initial release XMap is a fast network scanner designed for performing Internet-wide IPv6 & IPv4 network research scanning. --- .clang-format | 22 + .gitignore | 23 + .travis.yml | 42 + 10gigE.md | 59 + AUTHORS | 10 + CHANGELOG.md | 2 + CMakeLists.txt | 175 +++ CONTRIBUTING.md | 35 + Dockerfile | 34 + INSTALL.md | 94 ++ LICENSE | 201 +++ README.md | 53 + checkFormat.sh | 20 + conf/blacklist4.conf | 25 + conf/whitelist6.conf | 5 + conf/xmap.conf | 29 + examples/forge-socket/Makefile | 21 + examples/forge-socket/README.md | 73 + examples/forge-socket/forge-socket.c | 499 +++++++ examples/udp-probes/123.hex | 1 + examples/udp-probes/137.hex | 1 + examples/udp-probes/161.hex | 1 + examples/udp-probes/1900.txt | 8 + examples/udp-probes/520.hex | 1 + examples/udp-probes/53.hex | 1 + examples/udp-probes/README | 49 + examples/udp-probes/bacnet_rpm_47808.pkt | Bin 0 -> 37 bytes examples/udp-probes/chargen_19.pkt | 1 + examples/udp-probes/citrix_1604.pkt | Bin 0 -> 30 bytes examples/udp-probes/coap_5683.pkt | 1 + examples/udp-probes/db2disco_523.pkt | Bin 0 -> 20 bytes examples/udp-probes/digi1_2362.pkt | Bin 0 -> 14 bytes examples/udp-probes/digi2_2362.pkt | Bin 0 -> 14 bytes examples/udp-probes/digi3_2362.pkt | Bin 0 -> 14 bytes examples/udp-probes/dns_53.pkt | Bin 0 -> 30 bytes .../dns_53_queryAwww.google.com.pkt | Bin 0 -> 32 bytes .../udp-probes/dns_53_queryAwww.google.it.pkt | Bin 0 -> 31 bytes examples/udp-probes/ipmi_623.pkt | Bin 0 -> 23 bytes examples/udp-probes/ldap_389.pkt | Bin 0 -> 53 bytes examples/udp-probes/mdns_5353.pkt | Bin 0 -> 46 bytes examples/udp-probes/memcache_11211.pkt | Bin 0 -> 21 bytes examples/udp-probes/mssql_1434.pkt | 1 + examples/udp-probes/natpmp_5351.pkt | Bin 0 -> 4 bytes examples/udp-probes/netbios_137.pkt | Bin 0 -> 50 bytes examples/udp-probes/ntp_123.pkt | Bin 0 -> 48 bytes examples/udp-probes/ntp_123_monlist.pkt | Bin 0 -> 192 bytes examples/udp-probes/pca_nq_5632.pkt | 1 + examples/udp-probes/pca_st_5632.pkt | 1 + examples/udp-probes/portmap_111.pkt | Bin 0 -> 40 bytes examples/udp-probes/qotd_17.pkt | 1 + examples/udp-probes/ripv1_520.pkt | Bin 0 -> 24 bytes examples/udp-probes/sentinel_5093.pkt | Bin 0 -> 6 bytes examples/udp-probes/sip_5060.pkt | 1 + examples/udp-probes/sip_options.tpl | 12 + examples/udp-probes/snmp1_161.pkt | Bin 0 -> 43 bytes examples/udp-probes/snmp2_161.pkt | Bin 0 -> 40 bytes examples/udp-probes/snmp3_161.pkt | Bin 0 -> 60 bytes examples/udp-probes/ssdp_1900.pkt | 5 + examples/udp-probes/tftp_69.pkt | Bin 0 -> 14 bytes .../ubiquiti_discovery_v1_10001.pkt | Bin 0 -> 4 bytes .../ubiquiti_discovery_v2_10001.pkt | Bin 0 -> 4 bytes examples/udp-probes/upnp_1900.pkt | 7 + examples/udp-probes/wdbrpc_17185.pkt | Bin 0 -> 64 bytes examples/udp-probes/wsd_3702.pkt | 3 + examples/udp-probes/wsd_malformed_3702.pkt | 1 + examples/udp-probes/xdmcp_177.pkt | Bin 0 -> 7 bytes format.sh | 24 + lib/CMakeLists.txt | 28 + lib/blocklist.c | 485 +++++++ lib/blocklist.h | 56 + lib/bloom.c | 554 +++++++ lib/bloom.h | 212 +++ lib/constraint.c | 601 ++++++++ lib/constraint.h | 72 + lib/csv.c | 37 + lib/csv.h | 10 + lib/gmp-ext.c | 94 ++ lib/gmp-ext.h | 54 + lib/includes.h | 49 + lib/lockfd.c | 42 + lib/lockfd.h | 14 + lib/logger.c | 271 ++++ lib/logger.h | 63 + lib/random.c | 32 + lib/random.h | 9 + lib/redis.c | 429 ++++++ lib/redis.h | 91 ++ lib/rijndael-alg-fst.c | 1270 +++++++++++++++++ lib/rijndael-alg-fst.h | 57 + lib/types.h | 19 + lib/util.c | 454 ++++++ lib/util.h | 62 + lib/xalloc.c | 40 + lib/xalloc.h | 14 + scripts/README.md | 6 + scripts/check_manfile.py | 33 + scripts/cmake_install_checksum | 1 + scripts/install_cmake.py | 50 + scripts/install_jsonc.py | 46 + scripts/install_mongo.py | 38 + scripts/json_c_new_Makefile.am.inc | 1 + scripts/uninstall.cmake | 24 + src/CMakeLists.txt | 239 ++++ src/CMakeVersion.txt | 11 + src/aesrand.c | 78 + src/aesrand.h | 30 + src/cyclic.c | 636 +++++++++ src/cyclic.h | 57 + src/expression.c | 195 +++ src/expression.h | 49 + src/fieldset.c | 354 +++++ src/fieldset.h | 151 ++ src/filter.c | 109 ++ src/filter.h | 23 + src/get_gateway-bsd.h | 237 +++ src/get_gateway-linux.h | 345 +++++ src/get_gateway.c | 21 + src/get_gateway.h | 29 + src/iid_modules/iid_modules.c | 44 + src/iid_modules/iid_modules.h | 44 + src/iid_modules/module_full.c | 51 + src/iid_modules/module_low.c | 52 + src/iid_modules/module_low_fill.c | 53 + src/iid_modules/module_rand.c | 67 + src/iid_modules/module_set.c | 65 + src/iid_modules/module_zero.c | 42 + src/ip_target_file.c | 135 ++ src/ip_target_file.h | 29 + src/iterator.c | 173 +++ src/iterator.h | 43 + src/lexer.l | 36 + src/monitor.c | 517 +++++++ src/monitor.h | 23 + src/output_modules/module_csv.c | 130 ++ src/output_modules/module_csv.h | 22 + src/output_modules/module_json.c | 152 ++ src/output_modules/module_json.h | 16 + src/output_modules/module_redis_csv.c | 222 +++ src/output_modules/module_redis_csv.h | 21 + src/output_modules/module_redis_packed.c | 130 ++ src/output_modules/module_redis_packed.h | 21 + src/output_modules/output_modules.c | 62 + src/output_modules/output_modules.h | 47 + src/parser.y | 140 ++ src/probe_modules/module_icmp6.h | 29 + src/probe_modules/module_icmp6_echo.c | 382 +++++ src/probe_modules/module_icmp6_echo_gw.c | 384 +++++ src/probe_modules/module_icmp6_echo_tmxd.c | 393 +++++ src/probe_modules/module_icmp_echo.c | 385 +++++ src/probe_modules/module_tcp6_syn.c | 226 +++ src/probe_modules/module_tcp_syn.c | 221 +++ src/probe_modules/module_udp.c | 1070 ++++++++++++++ src/probe_modules/module_udp.h | 104 ++ src/probe_modules/module_udp6.c | 1079 ++++++++++++++ src/probe_modules/module_udp6.h | 104 ++ src/probe_modules/packet.c | 209 +++ src/probe_modules/packet.h | 354 +++++ src/probe_modules/packet_icmp.c | 220 +++ src/probe_modules/packet_icmp.h | 19 + src/probe_modules/packet_icmp6.c | 89 ++ src/probe_modules/packet_icmp6.h | 19 + src/probe_modules/probe_modules.c | 144 ++ src/probe_modules/probe_modules.h | 93 ++ src/recv-internal.h | 27 + src/recv-pcap.c | 156 ++ src/recv-pfring.c | 86 ++ src/recv.c | 215 +++ src/recv.h | 21 + src/send-bsd.h | 35 + src/send-linux.h | 71 + src/send-pfring.h | 46 + src/send.c | 505 +++++++ src/send.h | 24 + src/shard.c | 281 ++++ src/shard.h | 73 + src/socket-bsd.c | 67 + src/socket-linux.c | 40 + src/socket-pfring.c | 25 + src/socket.c | 33 + src/socket.h | 42 + src/state.c | 120 ++ src/state.h | 254 ++++ src/summary.c | 431 ++++++ src/summary.h | 19 + src/tests/xtest.c | 36 + src/utility.c | 153 ++ src/utility.h | 21 + src/validate.c | 53 + src/validate.h | 25 + src/xmap.1 | 353 +++++ src/xmap.1.html | 259 ++++ src/xmap.1.ronn | 279 ++++ src/xmap.c | 1197 ++++++++++++++++ src/xopt.ggo.in | 223 +++ src/xopt_include.c | 20 + 195 files changed, 22476 insertions(+) create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 10gigE.md create mode 100644 AUTHORS create mode 100644 CHANGELOG.md create mode 100644 CMakeLists.txt create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 INSTALL.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 checkFormat.sh create mode 100644 conf/blacklist4.conf create mode 100644 conf/whitelist6.conf create mode 100644 conf/xmap.conf create mode 100644 examples/forge-socket/Makefile create mode 100644 examples/forge-socket/README.md create mode 100644 examples/forge-socket/forge-socket.c create mode 100755 examples/udp-probes/123.hex create mode 100755 examples/udp-probes/137.hex create mode 100755 examples/udp-probes/161.hex create mode 100644 examples/udp-probes/1900.txt create mode 100755 examples/udp-probes/520.hex create mode 100755 examples/udp-probes/53.hex create mode 100644 examples/udp-probes/README create mode 100644 examples/udp-probes/bacnet_rpm_47808.pkt create mode 100644 examples/udp-probes/chargen_19.pkt create mode 100644 examples/udp-probes/citrix_1604.pkt create mode 100644 examples/udp-probes/coap_5683.pkt create mode 100644 examples/udp-probes/db2disco_523.pkt create mode 100644 examples/udp-probes/digi1_2362.pkt create mode 100644 examples/udp-probes/digi2_2362.pkt create mode 100644 examples/udp-probes/digi3_2362.pkt create mode 100644 examples/udp-probes/dns_53.pkt create mode 100644 examples/udp-probes/dns_53_queryAwww.google.com.pkt create mode 100644 examples/udp-probes/dns_53_queryAwww.google.it.pkt create mode 100644 examples/udp-probes/ipmi_623.pkt create mode 100644 examples/udp-probes/ldap_389.pkt create mode 100644 examples/udp-probes/mdns_5353.pkt create mode 100644 examples/udp-probes/memcache_11211.pkt create mode 100644 examples/udp-probes/mssql_1434.pkt create mode 100644 examples/udp-probes/natpmp_5351.pkt create mode 100644 examples/udp-probes/netbios_137.pkt create mode 100644 examples/udp-probes/ntp_123.pkt create mode 100644 examples/udp-probes/ntp_123_monlist.pkt create mode 100644 examples/udp-probes/pca_nq_5632.pkt create mode 100644 examples/udp-probes/pca_st_5632.pkt create mode 100644 examples/udp-probes/portmap_111.pkt create mode 100644 examples/udp-probes/qotd_17.pkt create mode 100644 examples/udp-probes/ripv1_520.pkt create mode 100644 examples/udp-probes/sentinel_5093.pkt create mode 100644 examples/udp-probes/sip_5060.pkt create mode 100644 examples/udp-probes/sip_options.tpl create mode 100644 examples/udp-probes/snmp1_161.pkt create mode 100644 examples/udp-probes/snmp2_161.pkt create mode 100644 examples/udp-probes/snmp3_161.pkt create mode 100644 examples/udp-probes/ssdp_1900.pkt create mode 100644 examples/udp-probes/tftp_69.pkt create mode 100644 examples/udp-probes/ubiquiti_discovery_v1_10001.pkt create mode 100644 examples/udp-probes/ubiquiti_discovery_v2_10001.pkt create mode 100644 examples/udp-probes/upnp_1900.pkt create mode 100644 examples/udp-probes/wdbrpc_17185.pkt create mode 100644 examples/udp-probes/wsd_3702.pkt create mode 100644 examples/udp-probes/wsd_malformed_3702.pkt create mode 100644 examples/udp-probes/xdmcp_177.pkt create mode 100644 format.sh create mode 100644 lib/CMakeLists.txt create mode 100644 lib/blocklist.c create mode 100644 lib/blocklist.h create mode 100644 lib/bloom.c create mode 100644 lib/bloom.h create mode 100644 lib/constraint.c create mode 100644 lib/constraint.h create mode 100644 lib/csv.c create mode 100644 lib/csv.h create mode 100644 lib/gmp-ext.c create mode 100644 lib/gmp-ext.h create mode 100644 lib/includes.h create mode 100644 lib/lockfd.c create mode 100644 lib/lockfd.h create mode 100644 lib/logger.c create mode 100644 lib/logger.h create mode 100644 lib/random.c create mode 100644 lib/random.h create mode 100644 lib/redis.c create mode 100644 lib/redis.h create mode 100644 lib/rijndael-alg-fst.c create mode 100644 lib/rijndael-alg-fst.h create mode 100644 lib/types.h create mode 100644 lib/util.c create mode 100644 lib/util.h create mode 100644 lib/xalloc.c create mode 100644 lib/xalloc.h create mode 100644 scripts/README.md create mode 100755 scripts/check_manfile.py create mode 100644 scripts/cmake_install_checksum create mode 100644 scripts/install_cmake.py create mode 100644 scripts/install_jsonc.py create mode 100644 scripts/install_mongo.py create mode 100644 scripts/json_c_new_Makefile.am.inc create mode 100644 scripts/uninstall.cmake create mode 100644 src/CMakeLists.txt create mode 100644 src/CMakeVersion.txt create mode 100644 src/aesrand.c create mode 100644 src/aesrand.h create mode 100644 src/cyclic.c create mode 100644 src/cyclic.h create mode 100644 src/expression.c create mode 100644 src/expression.h create mode 100644 src/fieldset.c create mode 100644 src/fieldset.h create mode 100644 src/filter.c create mode 100644 src/filter.h create mode 100644 src/get_gateway-bsd.h create mode 100644 src/get_gateway-linux.h create mode 100644 src/get_gateway.c create mode 100644 src/get_gateway.h create mode 100644 src/iid_modules/iid_modules.c create mode 100644 src/iid_modules/iid_modules.h create mode 100644 src/iid_modules/module_full.c create mode 100644 src/iid_modules/module_low.c create mode 100644 src/iid_modules/module_low_fill.c create mode 100644 src/iid_modules/module_rand.c create mode 100644 src/iid_modules/module_set.c create mode 100644 src/iid_modules/module_zero.c create mode 100644 src/ip_target_file.c create mode 100644 src/ip_target_file.h create mode 100644 src/iterator.c create mode 100644 src/iterator.h create mode 100644 src/lexer.l create mode 100644 src/monitor.c create mode 100644 src/monitor.h create mode 100644 src/output_modules/module_csv.c create mode 100644 src/output_modules/module_csv.h create mode 100644 src/output_modules/module_json.c create mode 100644 src/output_modules/module_json.h create mode 100644 src/output_modules/module_redis_csv.c create mode 100644 src/output_modules/module_redis_csv.h create mode 100644 src/output_modules/module_redis_packed.c create mode 100644 src/output_modules/module_redis_packed.h create mode 100644 src/output_modules/output_modules.c create mode 100644 src/output_modules/output_modules.h create mode 100644 src/parser.y create mode 100644 src/probe_modules/module_icmp6.h create mode 100644 src/probe_modules/module_icmp6_echo.c create mode 100644 src/probe_modules/module_icmp6_echo_gw.c create mode 100644 src/probe_modules/module_icmp6_echo_tmxd.c create mode 100644 src/probe_modules/module_icmp_echo.c create mode 100644 src/probe_modules/module_tcp6_syn.c create mode 100644 src/probe_modules/module_tcp_syn.c create mode 100644 src/probe_modules/module_udp.c create mode 100644 src/probe_modules/module_udp.h create mode 100644 src/probe_modules/module_udp6.c create mode 100644 src/probe_modules/module_udp6.h create mode 100644 src/probe_modules/packet.c create mode 100644 src/probe_modules/packet.h create mode 100644 src/probe_modules/packet_icmp.c create mode 100644 src/probe_modules/packet_icmp.h create mode 100644 src/probe_modules/packet_icmp6.c create mode 100644 src/probe_modules/packet_icmp6.h create mode 100644 src/probe_modules/probe_modules.c create mode 100644 src/probe_modules/probe_modules.h create mode 100644 src/recv-internal.h create mode 100644 src/recv-pcap.c create mode 100644 src/recv-pfring.c create mode 100644 src/recv.c create mode 100644 src/recv.h create mode 100644 src/send-bsd.h create mode 100644 src/send-linux.h create mode 100644 src/send-pfring.h create mode 100644 src/send.c create mode 100644 src/send.h create mode 100644 src/shard.c create mode 100644 src/shard.h create mode 100644 src/socket-bsd.c create mode 100644 src/socket-linux.c create mode 100644 src/socket-pfring.c create mode 100644 src/socket.c create mode 100644 src/socket.h create mode 100644 src/state.c create mode 100644 src/state.h create mode 100644 src/summary.c create mode 100644 src/summary.h create mode 100644 src/tests/xtest.c create mode 100644 src/utility.c create mode 100644 src/utility.h create mode 100644 src/validate.c create mode 100644 src/validate.h create mode 100644 src/xmap.1 create mode 100644 src/xmap.1.html create mode 100644 src/xmap.1.ronn create mode 100644 src/xmap.c create mode 100644 src/xopt.ggo.in create mode 100644 src/xopt_include.c diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..a43cff9 --- /dev/null +++ b/.clang-format @@ -0,0 +1,22 @@ +BasedOnStyle: LLVM +PointerAlignment: Right +IndentWidth: 4 +MaxEmptyLinesToKeep: 1 +ObjCSpaceAfterProperty: true +ObjCBlockIndentWidth: 4 +AllowShortFunctionsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: true +AlignTrailingComments: true +SpacesInSquareBrackets: false +SpacesInParentheses : false +AlignConsecutiveDeclarations: true +AlignConsecutiveAssignments: true +SpaceBeforeAssignmentOperators: true +SpacesInContainerLiterals: true +IndentWrappedFunctionNames: true +KeepEmptyLinesAtTheStartOfBlocks: true +BreakConstructorInitializersBeforeComma: true +AllowAllParametersOfDeclarationOnNextLine: true +SpaceAfterCStyleCast: true +TabWidth: 4 +UseTab: Never diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b636ae0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ + +.idea +.gitattributes +.DS_Store +cmake-build-debug +cmake_install.cmake +CMakeCache.txt +CMakeFiles +CPack* +Makefile +!examples/forge-socket/Makefile +lib/libxmaplib.a +src/lexer.c +src/lexer.h +src/parser.c +src/parser.h +src/xmap +src/xopt.c +src/xopt.ggo +src/xopt.h +install_manifest.txt +Testing +*.cbp diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..45f687b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,42 @@ +language: c +os: linux +dist: xenial +addons: + apt: + packages: + - build-essential + - cmake + - libgmp3-dev + - gengetopt + - libpcap-dev + - flex + - byacc + - libjson-c-dev + - pkg-config + - libunistring-dev + - libsasl2-dev + - python-dev + - python-pip + - libhiredis-dev +before_install: + - pip install --user sh + - export PATH=/opt/bin:$PATH +before_script: + - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - + - echo "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-6.0 main" | sudo tee -a /etc/apt/sources.list + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get update -qq + - sudo apt-get install libstdc++-4.9-dev libllvm6.0 clang-format-6.0 +script: + - mkdir __build + - cd __build + - cmake -DENABLE_DEVELOPMENT=ON -DWITH_JSON=ON -DWITH_REDIS=ON $TRAVIS_BUILD_DIR + - make + - cd $TRAVIS_BUILD_DIR + - python ./scripts/check_manfile.py +compiler: clang +notifications: + email: + - idealeer521@gmail.com + slack: + secure: WaFUBQz19oVEZL5lu3+GCU9q/e45POGYgH27jlbrBOtIUebIey219oSDC2G+5sG1L547nVlkrVa95L7C6sT5RP3D6aT8gWI0aDubaGjp4Wc1crau0LepxfRwpLOG/cYhQsaG31R/c5a3JIvgasV9P70bPyY9fhKbVxD7LbkWF7v7GB3jaJMjBZHHqWCXuzBoqowkqZARI13AAD7wbJJpkw8C/Tkw0xx1BvQsH+5DpPgd/0UfHBMotT4SMP2oHWUAjwL1KaxUxn/a4IpYTxMp4vHrTmtj7GxsQGPf2QQ80f2s5fCzd5veyqZfo3sMRbu/LgomwHTPsIhK8l3t4EXwSuxxmwQFqrEl+wQ0qM764cgKmIoqoeHMPt9RBQl9x7j8wPw46n5S/bNpTqAS2w3cuqpkkVqdD4wEST9rg4p4ERnJgvLvmdxAHFzl72qVwXwb+KhTEzdDgS/1OjdBGJDjLe6tBFdvCrNd6LQHx3NfDCxbQG3FyALIGMDsxDP3SLS0M8zkkCkEeShIrGDlshQQc8dL3YjTY08euTauaC6Esm6fZy4dWRM9EuRwYXFwwpyXTs/HWkIm+DUEk1IU2/m4YLb5IXnTQ+wjAPB2+oZqIZPDK6izo6tE3tCWWZuCr67Z8BQsM3oIAY98LDy3Nu34jJOuOjq0fwEoSEEgNoeB9zc= diff --git a/10gigE.md b/10gigE.md new file mode 100644 index 0000000..a1e8ddd --- /dev/null +++ b/10gigE.md @@ -0,0 +1,59 @@ +10GigE (Zippier) XMap +=========== + +It is possible to build XMap to run at 95% of 10 GigE linespeed, sending over 14 +million packets per second. This requires a compatible Intel 10 Gbps Ethernet +NIC and Linux. + +### Prerequisites + +0. A working XMap development environment (see [INSTALL.md](https://github.com/idealeer/xmap/blob/master/README.md)) + 1. A [PF_RING ZC](http://www.ntop.org/products/pf_ring/pf_ring-zc-zero-copy/) license from ntop. +2. PF_RING ZC headers and kernel module +3. A 10 Gbps NIC with compatible "PF_RING-aware" drivers +4. A Linux (not BSD or Mac) installation +4. For best results, a computer with at least 8 *physical* cores on the same NUMA node. +6. libnuma (`sudo apt-get install libnuma-dev`) + +### PF_RING Installation + +Coming soon. + +### Building + +Most build errors are due to incorrectly building or installing PF_RING. Make +sure you have build the drivers, the kernel module, and the userland library, as +well as install the headers and kernel module to the correct locations. + +The PF_RING `make install` command might not copy `pfring_zc.h` to +`/usr/include`, in which case manually install the file and set permissions +correctly. + +To build navigate to the root of the repository and run: + +``` +$ cmake -DWITH_PFRING=ON -DENABLE_DEVELOPMENT=OFF . +$ make +``` + +### Running + +You'll have to carefully select the number of threads to use, as well as specify +as zero-copy interface, e.g. `zc:eth1`. Use the `--cores` option to pick which +cores to pin to. Make sure to pin to different physical cores, and note that +some machines interleave physical and "virtual" cores. +``` +$ sudo ./src/xmap -p 80 -i zc:eth7 -o output.csv -T 5 +``` + +### Considerations + +**DO NOT TAKE THIS LIGHTLY!** + +Running XMap at 10Gbps hits every IPv4 prefix /16 on the Internet over 200 times a second. +Even if you have a large source IP range to scan from, it's very obvious that +you're scanning. As always, follow scanning best practices, honor blocklist +requests, and signal benign/research intent via domain names and websites on +your scan IPs. + +Remember, you're sending a lot of traffic. diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..b04f716 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,10 @@ +Zakir Durumeric +David Adrian +Eric Wustrow +J. Alex Halderman +Paul Pearce +Ariana Mirian +HD Moore +Oliver Gasser +Quirin Scheitle +Xiang Li diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b1b5815 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,2 @@ +# 1.0.0 2021-4-2 +* Initial public release. diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..6701084 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,175 @@ +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +project(XMAP C) +set(XMAP_VERSION DEVELOPMENT) # Change DEVELOPMENT to version number for release + +option(WITH_JSON "Build with support for JSON" ON) +option(WITH_REDIS "Build with support for Redis DB" OFF) +option(ENABLE_DEVELOPMENT "Enable development specific compiler and linker flags" OFF) +option(RESPECT_INSTALL_PREFIX_CONFIG "Respect CMAKE_INSTALL_PREFIX for /etc" OFF) +option(WITH_WERROR "Build with -Werror" OFF) +option(WITH_PFRING "Build with PF_RING ZC for send (10 GigE)" OFF) +option(ENABLE_LOG_TRACE "Enable log trace messages" OFF) +option(FORCE_CONF_INSTALL "Overwrites existing configuration files at install" OFF) + +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(USING_CLANG "YES") +else () + set(USING_GCC "YES") +endif () + +if ("${CMAKE_SYSTEM_NAME}" MATCHES "FreeBSD" OR "${CMAKE_SYSTEM_NAME}" MATCHES "NetBSD" OR "${CMAKE_SYSTEM_NAME}" MATCHES "DragonFly") + set(BSD "YES") +endif () + +if ("${CMAKE_SYSTEM_NAME}" MATCHES "NetBSD") + set(NetBSD "YES") +endif () + +# Hardening and warnings for building with gcc +# Maybe add -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations +set(GCCWARNINGS + "-Wall -Wformat=0 -Wno-format-nonliteral" + "-pedantic -fno-strict-aliasing" + "-Wextra" + "-Wfloat-equal -Wundef -Wwrite-strings -Wredundant-decls" + "-Wnested-externs -Wbad-function-cast -Winit-self" + "-Wmissing-noreturn" + "-Wstack-protector" + ) + +# Fix line breaks +string(REPLACE ";" " " GCCWARNINGS "${GCCWARNINGS}") + +if (WITH_WERROR) + set(GCCWARNINGS "${GCCWARNINGS} -Werror") +endif () + +if (ENABLE_DEVELOPMENT) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ggdb") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -g") +else () + # Hardening and optimizations for building with gcc + set(GCCHARDENING "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-all -fwrapv -fPIC --param ssp-buffer-size=1") + if (NOT APPLE AND NOT BSD) + set(LDHARDENING "-z relro -z now") + else () + set(LDHARDENING "") + endif () + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCCHARDENING} -O2") + set(CMAKE_EXE_LINKER_FLAGS "${LDHARDENING} ${CMAKE_EXE_LINKER_FLAGS}") +endif () + +set(CMAKE_C_FLAGS "${GCCWARNINGS} ${CMAKE_C_FLAGS}") + +if (ENABLE_LOG_TRACE) + add_definitions("-DDEBUG") +endif () + +if (WITH_JSON) + include(FindPkgConfig) + pkg_check_modules(JSON json-c) + if (JSON_FOUND) + include_directories(${JSON_INCLUDE_DIRS}) + else () + message(FATAL_ERROR "Did not find libjson") + endif () + add_definitions("-DJSON") + string(REPLACE ";" " " JSON_CFLAGS "${JSON_CFLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${JSON_CFLAGS}") +endif () + +if (WITH_REDIS) + set(REDIS_LIBS hiredis) + add_definitions("-DREDIS") +endif () + +if (WITH_PFRING) + add_definitions("-DPFRING") + set(PFRING_LIBRARIES pfring rt numa) +endif () + +# Standard FLAGS +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") +if (NOT APPLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") +endif () + +# Set up OS-specific include directories +if (APPLE) + if (EXISTS /opt/local/include) + include_directories(/opt/local/include) + endif () + if (EXISTS /opt/local/lib) + link_directories(/opt/local/lib) + endif () + if (EXISTS /usr/local/include) + include_directories(/usr/local/include) + endif () + if (EXISTS /usr/local/lib) + link_directories(/usr/local/lib) + endif () +endif () + +if (BSD) + include_directories(/usr/local/include) + link_directories(/usr/local/lib) +endif () + +if (NetBSD) + include_directories(/usr/pkg/include) + link_directories(/usr/pkg/lib) +endif () + +add_subdirectory("lib") +add_subdirectory("src") + +# Install conf files +if (RESPECT_INSTALL_PREFIX_CONFIG) + set(CONFIG_DESTINATION "etc/xmap") +else () + set(CONFIG_DESTINATION "/etc/xmap") +endif () + +FILE(GLOB CONF_FILES "${PROJECT_SOURCE_DIR}/conf/*") + +message(STATUS "Default XMap configuration file location is /etc/xmap") +foreach (EACH_CONF ${CONF_FILES}) + get_filename_component(CONF_BASENAME ${EACH_CONF} NAME) + message(STATUS "Checking if ${CONF_BASENAME} exists there...") + if (NOT EXISTS "/etc/xmap/${CONF_BASENAME}") + install(FILES ${EACH_CONF} DESTINATION ${CONFIG_DESTINATION}) + elseif (FORCE_CONF_INSTALL) + message(WARNING "FORCE_CONF_INSTALL will overwrite any exsiting configuration files") + install(FILES ${EACH_CONF} DESTINATION ${CONFIG_DESTINATION}) + else () + message(WARNING "Existing configuration file detected at /etc/xmap/${CONF_BASENAME}, ${CONF_BASENAME} from sources will NOT be installed. Please check and install manually!") + endif () +endforeach () + +# Uninstall +add_custom_target(uninstall + ${CMAKE_COMMAND} -P "${CMAKE_SOURCE_DIR}/scripts/uninstall.cmake" + ) + +# Allow Debian Packaging +include(InstallRequiredSystemLibraries) + +set(CPACK_SET_DESTDIR "on") +set(CPACK_PACKAGING_INSTALL_PREFIX "/tmp") +set(CPACK_GENERATOR "DEB") + +set(${VERSION} CPACK_DEBIAN_PACKAGE_VERSION) +set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional") +set(CPACK_DEBIAN_PACKAGE_SECTION "network") +set(CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.1.3), libgmp10, libpcap0.8, libjson-c-dev") + +set(CPACK_PACKAGE_DESCRIPTION "Internet-wide IPv6 & IPv4 Network Scanner") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "XMap is an open-source network scanner that enables researchers to perform Internet-wide IPv6 & IPv4 network research scanning efficiently. XMap is reimplemented and improved thoroughly from ZMap and is fully compatible with ZMap, armed with the " 5 minutes" probing speed and novel IPv6 scanning techniques. XMap can be used to conduct large-scale IPv6 network scanning for device discovery and security assessments and help us gain more insights into the IPv6 networks.") +set(CPACK_PACKAGE_CONTACT "Xiang Li ") +set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${VERSION}_${CPACK_DEBIAN_ARCHITECTURE}") + +set(CPACK_COMPONENTS_ALL Libraries ApplicationData) + +include(CPack) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d62f85b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,35 @@ +Contributing to XMap +==================== + +XMap accepts contributions in the form of issues and pull requests. In either +case, before posting please [search](https://github.com/idealeer/xmap/issues) to see +if your change or bug report has been addressed previously. + +[INSTALL](INSTALL.md#building-from-source) provides guidance on building XMap from source. + +Developing +---------- + +- XMap code follows the [Linux kernel style guide][kernelguide]. We mantain [a + configuration file](/.clang-format) for `clang-format` that applies this + style. You can use the [format.sh](/format.sh) script to apply this style. + +- Before submitting a PR, please rebase/squash your commits down to a single + commit. Follow these [commit message guidelines][guidelines], especially with + regard to formatting. + +Reviewing +--------- + +- All commits must be reviewed in the form of a pull request by a XMap + maintainer. This usually means @idealeer. + +- All pull-requests should be squash-merged into master. + +- When squash-merging, put the PR number in the commit title. GitHub does this + automatically in the web interface. Condense the commit messages down to a + single message; often this can just be the commit message from the first + commit in a PR. Follow the commit formatting guidelines [here][guidelines]. + +[kernelguide]: https://www.kernel.org/doc/Documentation/process/coding-style.rst +[guidelines]: https://github.com/torvalds/subsurface-for-dirk/blob/master/README#L92 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b9d0c71 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +#### +# A Docker container for running xmap +# +# To build, beware of caching and: +# +# * If you wish to build current master +# +# docker build -t xmap_ubuntu -f Dockerfile . +# +# * If you wish to build a specific commit, use the XMAP_COMMIT build argument. +# +# docker build -t xmap_ubuntu -f Dockerfile --build-arg XMAP_COMMIT= . +# +# To run: +# +# docker run -it --rm --net=host xmap_ubuntu +#### + +FROM ubuntu:16.04 + +ARG XMAP_COMMIT=master +ENV XMAP_COMMIT ${XMAP_COMMIT} + +RUN apt-get -qq update && apt-get -qqy upgrade +# install xmap build dependencies +RUN apt-get -qqy install build-essential cmake libgmp3-dev gengetopt libpcap-dev flex byacc libjson-c-dev pkg-config libunistring-dev wget unzip +# install xmap+Docker specific things, currently just dumb-init, which allows +# us to more easily send signals to xmap, for example by allowing ctrl-c of +# a running container and xmap will stop. +RUN apt-get -qqy install python-dev python-pip +RUN pip install dumb-init +RUN wget -q https://github.com/idealeer/xmap/archive/${XMAP_COMMIT}.zip && unzip -q ${XMAP_COMMIT}.zip && cd xmap-${XMAP_COMMIT} && (cmake . && make -j4 && make install) 2>&1 > /dev/null + +ENTRYPOINT ["dumb-init", "/usr/local/sbin/xmap"] diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..d448487 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,94 @@ +# Installing and Building XMap + +## Installing via Package Manager + +XMap operates on GNU/Linux, macOS, and BSD. + +**Installing with the most OS package managers is not integrated yet.** + +| OS | | +| ----------------------------------------- | ---- | +| Fedora 19+ or EPEL 6+ | `-` | +| Debian 8+ or Ubuntu 14.04+ | `-` | +| Gentoo | `-` | +| macOS (using [Homebrew](https://brew.sh)) | `-` | +| Arch Linux | `-` | + +## Building from Source + +### Installing XMap Dependencies + +XMap has the following dependencies: + +- [CMake](http://www.cmake.org/) - Cross-platform, open-source build system +- [GMP](http://gmplib.org/) - Free library for arbitrary precision arithmetic +- [gengetopt](http://www.gnu.org/software/gengetopt/gengetopt.html) - Command line option parsing for C programs +- [libpcap](http://www.tcpdump.org/) - Famous user-level packet capture library +- [flex](http://flex.sourceforge.net/) and [byacc](http://invisible-island.net/byacc/) - Output filter lexer and parser generator +- [json-c](https://github.com/json-c/json-c/) - JSON implementation in C +- [libunistring](https://www.gnu.org/software/libunistring/) - Unicode string library for C +- [libdnet](https://github.com/dugsong/libdnet) - (macOS Only) Gateway and route detection + +In addition, the following optional packages enable optional XMap functionality: + +- [hiredis](https://github.com/redis/hiredis) - RedisDB support in C + +Install the required dependencies with the following commands. + +* On Debian-based systems (including Ubuntu): + ```sh + sudo apt-get install build-essential cmake libgmp3-dev gengetopt libpcap-dev flex byacc libjson-c-dev pkg-config libunistring-dev + ``` + +* On RHEL- and Fedora-based systems (including CentOS): + ```sh + sudo yum install cmake gmp-devel gengetopt libpcap-devel flex byacc json-c-devel libunistring-devel + ``` + +* On macOS systems (using [Homebrew](http://brew.sh/)): + ```sh + brew install pkg-config cmake gmp gengetopt json-c byacc libdnet libunistring + ``` + +### Building and Installing XMap + +Once these prerequisites are installed, XMap can be compiled by running: + ```sh + cmake . + make -j4 + ``` + +### Development Notes + +- Enabling development turns on debug symbols, and turns off optimizations. + Release builds should be built with `-DENABLE_DEVELOPMENT=OFF`. + +- Enabling `log_trace` can have a major performance impact and should not be used + except during early development. Release builds should be built with `-DENABLE_LOG_TRACE=OFF`. + +- Redis support is not enabled by default. If you want to use XMap with Redis, + you will first need to install hiredis. Then run cmake with `-DWITH_REDIS=ON`. + Debian/Ubuntu has packaged hiredis as `libhiredis-dev`; Fedora and RHEL/CentOS + have packaged it as `hiredis-devel`. + +- Building packages for some systems like Fedora and RHEL requires a user-definable + directory (buildroot) to put files. The way to respect this prefix is to run cmake + with `-DRESPECT_INSTALL_PREFIX_CONFIG=ON`. + +- Manpages (and their HTML representations) are generated from the `.ronn` source + files in the repository, using the [ronn](https://github.com/rtomayko/ronn) tool. + This does not happen automatically as part of the build process; to regenerate the + man pages you'll need to run `make manpages`. This target assumes that `ronn` is + in your PATH. + +- Building with some versions of CMake may fail with `unable to find parser.h`. + If this happens, try updating CMake. If it still fails, don't clone XMap into a + path that contains the string `.com`, and try again. + +- XMap may be installed to an alternative directory, with the `CMAKE_INSTALL_PREFIX` + option. For example, to install it in `$HOME/opt` run + ```sh + cmake -DCMAKE_INSTALL_PREFIX=$HOME/opt . + make -j4 + make install + ``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b09cd78 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..d00b79b --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +XMap: The Internet Scanner +========================== +[![Build Status](https://travis-ci.com/idealeer/xmap.svg?token=Si5TyFph867jMev16gn1&branch=master)](https://travis-ci.com/idealeer/xmap) + +XMap is a fast network scanner designed for performing Internet-wide IPv6 & IPv4 network research scanning. + +XMap is reimplemented and improved thoroughly from ZMap and is fully compatible with ZMap, armed with the "5 minutes" probing speed and novel scanning techniques. XMap is capable of scanning the 32-bits address space in under 45 minutes. With a 10 gigE connection and [PF_RING](http://www.ntop.org/products/packet-capture/pf_ring/), XMap can scan the 32-bits address space in under 5 minutes. Moreover, leveraging the novel IPv6 scanning approach, XMap can discover the IPv6 Network Periphery fast. Furthermore, XMap can scan the network space randomly with any length and at any position, such as 2001:db8::/32-64 and 192.168.0.1/16-20. Besides, XMap can probe multiple ports simultaneously. + +XMap operates on GNU/Linux, Mac OS, and BSD. XMap currently has implemented probe modules for ICMP Echo scans, TCP SYN scans, and [UDP probes](https://github.com/idealeer/xmap/blob/master/examples/udp-probes/README). + +With banner grab and TLS handshake tool, [ZGrab2](https://github.com/zmap/zgrab2), more involved scans could be performed. + +Installation +------------ + +The latest stable release of XMap is version 1.0.0 and supports Linux, macOS, and BSD. We recommend installing XMap from HEAD rather than using a distro package manager (not supported yet). + +**Instructions on building XMap from source** can be found in [INSTALL](https://github.com/idealeer/xmap/blob/master/README.md). + +Usage +----- + +A guide to using XMap is found in our [GitHub Wiki](https://github.com/idealeer/xmap/wiki). + +## Paper + +Fast IPv6 Network Periphery Discovery and Security Implications. + +**Abstract.** IPv6 fundamentally changes address allocation principles, making brute-force scanning of the 128-bits IPv6 network space impractical. Although significant efforts have been devoted to enumerating active IPv6 hosts, limited by technique efficiency and accuracy, large-scale empirical measurement studies under the IPv6 network are still infeasible now. + +To fill this research gap, by leveraging the extensively adopted IPv6 address allocation strategy, we propose a novel IPv6 network periphery discovery approach. Specifically, *XMap*, a fast network scanner, is developed to find the periphery, such as a home router. We evaluate it on twelve prominent Internet service providers and harvest *52M* active peripheries. Grounded on these found devices, we explore IPv6 network risks of the unintended exposed security services and the flawed traffic routing strategies. First, we demonstrate the unintended exposed security services in IPv6 networks, such as DNS, and HTTP, have become emerging security risks by analyzing *4.7M* peripheries. Second, by inspecting the periphery's packet routing strategies, we present the flawed implementations of IPv6 routing protocol affecting *5.8M* router devices. Attackers can exploit this common vulnerability to conduct effective routing loop attacks, inducing DoS to the ISP's and home routers with an amplification factor of *>200*. We responsibly disclose those issues to all involved vendors and ASes and discuss mitigation solutions. Our research results indicate that the security community should revisit IPv6 network strategies immediately. + +**Authors.** [Xiang Li](https://netsec.ccert.edu.cn/people/lx19), [Baojun Liu](https://netsec.ccert.edu.cn/people/lbj20/), Xiaofeng Zheng, [Haixin Duan](https://netsec.ccert.edu.cn/people/duanhx/), [Qi Li](https://netsec.ccert.edu.cn/people/qli/), Youjun Huang. + +**Conference.** Proceedings of the 2021 IEEE/IFIP International Conference on Dependable Systems and Networks (DSN '21) + +**Paper.** [Abstract], [PDF], [Slides] and [Video]. + +**CNVD/CVE.** [[Lists]](https://netsec.ccert.edu.cn/people/lx19). + +License and Copyright +--------------------- + +XMap Copyright 2021 Xiang Li from Network and Information Security Lab Tsinghua University + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See LICENSE for the specific +language governing permissions and limitations under the License. diff --git a/checkFormat.sh b/checkFormat.sh new file mode 100644 index 0000000..027cc3b --- /dev/null +++ b/checkFormat.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +CLANG_FORMAT=clang-format-6.0 + +files_to_lint=$(find ./src ./lib -type f -name '*.c' -or -name '*.h') + +fail=0 +for f in ${files_to_lint}; do + d="$(diff -u "$f" <($CLANG_FORMAT -style=file "$f") || true)" + if ! [ -z "$d" ]; then + printf "The file %s is not compliant with the coding style:\n%s\n" "$f" "$d" + fail=1 + fi +done + +if [ "$fail" -eq "1" ]; then + if [ ! -z $XMAP_ENFORCE_FORMAT ]; then + exit 1 + fi +fi diff --git a/conf/blacklist4.conf b/conf/blacklist4.conf new file mode 100644 index 0000000..018b0f4 --- /dev/null +++ b/conf/blacklist4.conf @@ -0,0 +1,25 @@ +# From IANA IPv4 Special-Purpose Address Registry +# http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +# Updated 2013-05-22 + +0.0.0.0/8 # RFC1122: "This host on this network" +10.0.0.0/8 # RFC1918: Private-Use +100.64.0.0/10 # RFC6598: Shared Address Space +127.0.0.0/8 # RFC1122: Loopback +169.254.0.0/16 # RFC3927: Link Local +172.16.0.0/12 # RFC1918: Private-Use +192.0.0.0/24 # RFC6890: IETF Protocol Assignments +192.0.2.0/24 # RFC5737: Documentation (TEST-NET-1) +192.88.99.0/24 # RFC3068: 6to4 Relay Anycast +192.168.0.0/16 # RFC1918: Private-Use +198.18.0.0/15 # RFC2544: Benchmarking +198.51.100.0/24 # RFC5737: Documentation (TEST-NET-2) +203.0.113.0/24 # RFC5737: Documentation (TEST-NET-3) +240.0.0.0/4 # RFC1112: Reserved +255.255.255.255/32 # RFC0919: Limited Broadcast + +# From IANA Multicast Address Space Registry +# http://www.iana.org/assignments/multicast-addresses/multicast-addresses.xhtml +# Updated 2013-06-25 + +224.0.0.0/4 # RFC5771: Multicast/Reserved diff --git a/conf/whitelist6.conf b/conf/whitelist6.conf new file mode 100644 index 0000000..54bfb54 --- /dev/null +++ b/conf/whitelist6.conf @@ -0,0 +1,5 @@ +# From IANA IPv6 Global Unicast Address Assignments +# https://www.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.xhtml +# Updated 2019-11-06 + +2000::/3 # RFC7249: "The IPv6 Global Unicast Address space" diff --git a/conf/xmap.conf b/conf/xmap.conf new file mode 100644 index 0000000..c6ff032 --- /dev/null +++ b/conf/xmap.conf @@ -0,0 +1,29 @@ +### Network +#ipv6 + +### Probe Module to use +#probe-module icmp_echo + + +### Destination port to scan +#target-port 80 + +### Scan rate in packets/sec +#rate 10000 + +### Scan rate in bandwidth (bits/sec); overrides `rate` +#bandwidth 1M # 1mbps + + +### Blacklist file to use. We encourage you to exclude +### RFC1918, IANA reserved, and multicast networks, +### in addition to those who have opted out of your +### network scans. +#blacklist-file "/etc/xmap/blacklist4.conf" + +### Whitelist file to use. We encourage you to include +### RFC7249, IANA IPv6 Global Unicast Address. +#whitelist-file "/etc/xmap/whitelist6.conf" + +### Optionally print a summary at the end +#summary diff --git a/examples/forge-socket/Makefile b/examples/forge-socket/Makefile new file mode 100644 index 0000000..387afe5 --- /dev/null +++ b/examples/forge-socket/Makefile @@ -0,0 +1,21 @@ + +CFLAGS+=-I../../lib/ -I../../forge_socket -Wall +LDFLAGS+=-lpcap -levent -levent_extra -lm +VPATH=../../lib/ + +# from dpkg-buildflags --get CFLAGS, but use stack-protector-all and fPIC +GCCHARDENING=-g -O2 -fstack-protector-all --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security -fPIC +# from gpkg-buildflags --get LDFLAGS, + -z,now +LDHARDENING=-Wl,-Bsymbolic-functions -Wl,-z,relro,-z,now + +CFLAGS+=$(GCCHARDENING) +LDFLAGS+=$(LDHARDENING) + + +all: forge-socket + +forge-socket: forge-socket.o xalloc.o logger.o + $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS) + +clean: + rm -f forge-socket *.o diff --git a/examples/forge-socket/README.md b/examples/forge-socket/README.md new file mode 100644 index 0000000..43e3417 --- /dev/null +++ b/examples/forge-socket/README.md @@ -0,0 +1,73 @@ +Forge-socket banner grab +====== + +This utility, in combination with a kernel module +(https://github.com/ewust/forge_socket/) will complete the half-open connection +created by XMap during a TCP-scan, optionally send a small message, and wait +for the hosts response. The response is then printed along with their IP +address on stdout. Periodic status messages appear on stderr. + +This utility is functionally equivalent to banner-grab-tcp, however, instead of +having the kernel send a RST packet for the server's SYN+ACK, and +banner-grab-tcp attempting to start a fresh TCP connection with the host, +forge-socket will take the parameters of the SYN+ACK packet, and use a kernel +module to add it as an ESTABLISHED TCP connection socket. Then, the +forge-socket user-space program can use this socket to send() and recv() as +normal, and completes the banner-grab process (optionally send a small message, +and receive the server's response). + + + +USING: +----- +# Install forge-socket to the XMap root directory: +cd ./xmap/ +git clone git@github.com:ewust/forge_socket.git +cd forge_socket +make +sudo insmod forge_socket.ko + +# Don't send RST packets (forge-socket will complete these connections instead) +sudo iptables -A OUTPUT -p tcp -m tcp --tcp-flags RST,RST RST,RST -j DROP + +# Use XMap + forge-socket simultaneously: +make +#echo -e -n "GET / HTTP/1.1\r\nHost: %s\r\n\r\n" > http-req +sudo su +ulimit -SHn 1000000 && ulimit -SSn 1000000 +xmap -4 -p 80 -B 50M -N 1000 -O extended_file -o - | ./forge-socket -c 8000 -d http-req > http-banners.out + + +The options are similar to banner-grab-tcp, except there is no connection timeout :) + +OPTIONS: +----- +-c, --concurent Number of connections that can be going on at once. + This, combined with timeouts, will decide the maximum + rate at which banners are grabbed. If this value + is set higher than 1000, you should use + `ulimit -SSn 1000000` and `ulimit -SHn 1000000` to + avoid running out of file descriptors (typically capped + at 1024). + +-r, --read-timeout Read timeout (seconds). Give up on a host if after + connecting (and optionally sending data), it does + not send any response by this time. Default: 4 seconds. + +-v, --verbosity Set status verbosity. Status/error messages are outputed + on stderr. This value can be 0-5, with 5 being the most + verbose (LOG_TRACE). Default: 3 (LOG_INFO) + +-f, --format Format to output banner responses. One of 'hex', 'ascii', + or 'base64'. + 'hex' outputs ascii hex characters, e.g. 48656c6c6f. + 'ascii' outputs ascii, without separators, e.g. Hello + 'base64' outputs base64 encoding, e.g. SGVsbG8= + Default is base64. + +-d, --data Optional data file. This data will be sent to each host + upon successful connection. Currently, this file does + not allow null characters, but supports up to 4 + occurances of the current host's IP address, by replacing + %s with the string (inet_ntoa) of that host's IP address. + diff --git a/examples/forge-socket/forge-socket.c b/examples/forge-socket/forge-socket.c new file mode 100644 index 0000000..0e411fd --- /dev/null +++ b/examples/forge-socket/forge-socket.c @@ -0,0 +1,499 @@ +/* + * Forge Socket Banner Grab Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "logger.h" +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "forge_socket.h" + +#define MAX_BANNER_LEN 1024 +#define BASE64_ALPHABET \ + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +struct config { + int read_timeout; // how long to wait once connected for the banner + // (seconds) + int current_running; + int max_concurrent; + struct event_base * base; + struct bufferevent *stdin_bev; + int stdin_closed; + enum { FORMAT_HEX, FORMAT_BASE64, FORMAT_ASCII } format; + char *send_str; + long send_str_size; + + struct stats_st { + int init_connected_hosts; // Number of hosts we have even tried to + // connect to + int connected_hosts; // # hosts that picked up + int conn_timed_out; // # hosts that timed out during connection + int read_timed_out; // # hosts that connected, but sent no data (banner) + int timed_out; // # hosts that timed out at all + // (conn_timed_out+read_timed_out)? + int completed_hosts; // # hosts that presented a banner + } stats; +}; + +struct state { + struct config *conf; + uint32_t src_ip; + uint32_t dst_ip; + uint16_t sport; + uint16_t dport; + uint32_t seq; + uint32_t seq_ack; + enum { CONNECTING, CONNECTED, RECEIVED } state; +}; + +void stdin_readcb(struct bufferevent *bev, void *arg); + +void print_status(evutil_socket_t fd, short events, void *arg) { + struct event * ev; + struct config * conf = arg; + struct event_base *base = conf->base; + struct timeval status_timeout = {1, 0}; + ev = evtimer_new(base, print_status, conf); + evtimer_add(ev, &status_timeout); + (void) fd; + (void) events; + + log_info("forge-socket", + "(%d/%d in use) - Totals: %d inited, %d connected, %d conn " + "timeout, %d read timeout %d completed", + conf->current_running, conf->max_concurrent, + conf->stats.init_connected_hosts, conf->stats.connected_hosts, + conf->stats.conn_timed_out, conf->stats.read_timed_out, + conf->stats.completed_hosts); +} + +void decrement_cur_running(struct state *st) { + struct config *conf = st->conf; + conf->current_running--; + log_debug("forge-socket", "done, down to %d", conf->current_running); + if (evbuffer_get_length(bufferevent_get_input(conf->stdin_bev)) > 0) { + stdin_readcb(conf->stdin_bev, conf); + } + free(st); + + if (conf->stdin_closed && conf->current_running == 0) { + // Done + log_info("forge-socket", "done"); + print_status(0, 0, conf); + exit(0); + } +} + +void event_cb(struct bufferevent *bev, short events, void *arg) { + struct state * st = arg; + struct config *conf = st->conf; + struct in_addr addr; + addr.s_addr = st->src_ip; + if (events & BEV_EVENT_CONNECTED) { + log_error("forge-socket", "%s connected - wat?", inet_ntoa(addr)); + + } else { + if (st->state == CONNECTED) { + // Print out that we just didn't receive data + printf("%s X\n", inet_ntoa(addr)); + fflush(stdout); + + conf->stats.read_timed_out++; + } else { + conf->stats.conn_timed_out++; + } + log_debug("forge-socket", "%s bailing..", inet_ntoa(addr)); + bufferevent_free(bev); + conf->stats.timed_out++; + decrement_cur_running(st); + } +} + +// Grab these bytes, and close the connection. +// Even if we don't need to read any bytes, +// we have to have this so that libevent thinks we have +// a read event, so that it can timeout TCP connects +// (as a read timeout) +void read_cb(struct bufferevent *bev, void *arg) { + struct evbuffer *in = bufferevent_get_input(bev); + struct state * st = arg; + size_t len = evbuffer_get_length(in); + struct in_addr addr; + addr.s_addr = st->src_ip; + + log_debug("forge-socket", "read_cb for %s", inet_ntoa(addr)); + + if (len > MAX_BANNER_LEN) { + len = MAX_BANNER_LEN; + } + + if (len > 0) { + // Grab the banner + unsigned int i; + unsigned char *buf = malloc(len + 1); + + st->state = RECEIVED; + + if (!buf) { + log_fatal("forge-socket", "cannot alloc %d byte buf", len + 1); + return; + } + evbuffer_remove(in, buf, len); + + printf("%s ", inet_ntoa(addr)); + + if (st->conf->format == FORMAT_ASCII) { + // Ascii + buf[len] = '\0'; + printf("%s\n", buf); + } else if (st->conf->format == FORMAT_HEX) { + // Hex output + for (i = 0; i < len; i++) { + printf("%02x", buf[i]); + } + printf("\n"); + } else if (st->conf->format == FORMAT_BASE64) { + // Base64 + int i = 0; + char out[4] = {0, 0, 0, 0}; + while (i < len) { + uint32_t value = 0; + value += (i < len) ? buf[i++] << 16 : 0; + value += (i < len) ? buf[i++] << 8 : 0; + value += (i < len) ? buf[i++] : 0; + out[0] = BASE64_ALPHABET[(value >> 18) & 0x3F]; + out[1] = BASE64_ALPHABET[(value >> 12) & 0x3F]; + out[2] = BASE64_ALPHABET[(value >> 6) & 0x3F]; + out[3] = BASE64_ALPHABET[(value) &0x3F]; + if (i < len) { + printf("%c%c%c%c", out[0], out[1], out[2], out[3]); + } + } + if (len > 0) { + switch (len % 3) { + case 1: + out[2] = '='; + case 2: + out[3] = '='; + default: + break; + } + printf("%c%c%c%c\n", out[0], out[1], out[2], out[3]); + } + } + fflush(stdout); + + free(buf); + st->conf->stats.completed_hosts++; + } + bufferevent_free(bev); + decrement_cur_running(st); +} + +int set_sock_state(int sock, struct tcp_state *st) { + struct sockaddr_in sin; + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = st->src_ip; + sin.sin_port = st->sport; + + int value = 1; + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(value)) < 0) { + perror("setsockopt SO_REUSEADDR"); + return -1; + } + + if (setsockopt(sock, SOL_IP, IP_TRANSPARENT, &value, sizeof(value)) < 0) { + perror("setsockopt IP_TRANSPARENT"); + return -1; + } + + if (bind(sock, (struct sockaddr *) &sin, sizeof(sin)) < 0) { + perror("bind"); + return -1; + } + + if (setsockopt(sock, IPPROTO_TCP, TCP_STATE, st, sizeof(struct tcp_state)) < + 0) { + perror("setsockopt TCP_STATE"); + return -1; + } + + return 0; +} + +void grab_banner(struct state *st) { + struct sockaddr_in addr; + struct bufferevent *bev; + struct timeval read_to = {st->conf->read_timeout, 0}; + struct tcp_state tcp_st; + int sock = socket(AF_INET, SOCK_FORGE, 0); + + addr.sin_addr.s_addr = st->src_ip; + + if (sock < 0) { + perror("SOCK_FORGE socket"); + log_fatal("forge_socket", "(did you insmod forge_socket.ko?)"); + return; + } + + memset(&tcp_st, 0, sizeof(tcp_st)); + + // These need to be in network order for forge socket" + tcp_st.src_ip = st->dst_ip; + tcp_st.dst_ip = st->src_ip; + tcp_st.sport = htons(st->dport); + tcp_st.dport = htons(st->sport); + + // This should be in ??? + tcp_st.seq = st->seq_ack; + tcp_st.ack = (st->seq + 1); + + tcp_st.snd_wnd = 0x1000; + tcp_st.rcv_wnd = 0x1000; + + tcp_st.snd_una = tcp_st.seq; + st->state = CONNECTING; + st->conf->stats.init_connected_hosts++; + + // consider this a non-blocking, but completed "connect()". heh. + if (set_sock_state(sock, &tcp_st) != 0) { + log_error("forge_socket", "set_sock_state failed\n"); + decrement_cur_running(st); + return; + } + + evutil_make_socket_nonblocking(sock); + + bev = bufferevent_socket_new(st->conf->base, sock, BEV_OPT_CLOSE_ON_FREE); + + bufferevent_set_timeouts(bev, &read_to, &read_to); + + bufferevent_setcb(bev, read_cb, NULL, event_cb, st); + bufferevent_enable(bev, EV_READ); + + // Send data + if (st->conf->send_str) { + struct evbuffer *evout = bufferevent_get_output(bev); + // HACK!!! TODO: make some messy parser that replaces ${IP} with IP etc + // and allow null characters + evbuffer_add_printf(evout, st->conf->send_str, inet_ntoa(addr.sin_addr), + inet_ntoa(addr.sin_addr), inet_ntoa(addr.sin_addr), + inet_ntoa(addr.sin_addr)); + log_trace("forge-socket", "sent str to %s", inet_ntoa(addr.sin_addr)); + } + + // Update state/stats + st->state = CONNECTED; + st->conf->stats.connected_hosts++; + + log_trace("forge-socket", "go %s go! read a byte!!", + inet_ntoa(addr.sin_addr)); +} + +void stdin_eventcb(struct bufferevent *bev, short events, void *ptr) { + struct config *conf = ptr; + + if (events & BEV_EVENT_EOF) { + log_debug("forge-socket", + "received EOF; quitting after buffer empties"); + conf->stdin_closed = 1; + if (conf->current_running == 0) { + log_info("forge-socket", "done"); + print_status(0, 0, conf); + exit(0); + } + } +} + +void stdin_readcb(struct bufferevent *bev, void *arg) { + struct evbuffer *in = bufferevent_get_input(bev); + struct config * conf = arg; + + log_debug("forge-socket", "stdin cb %d < %d ?", conf->current_running, + conf->max_concurrent); + + while (conf->current_running < conf->max_concurrent && + evbuffer_get_length(in) > 0) { + size_t line_len; + char * line = evbuffer_readln(in, &line_len, EVBUFFER_EOL_LF); + struct state *st; + if (!line) break; + log_debug("forge-socket", "line: '%s'", line); + + // synack, 77.176.116.205, 141.212.121.125, 443, 49588, 3628826326, + // 3441755636, 0, 0,2013-08-11 19:16:05.799 + char synack[12]; + char srcip[INET_ADDRSTRLEN], dstip[INET_ADDRSTRLEN]; + uint32_t seq, seq_ack; + uint16_t sport, dport; + int cooldown, repeat = 1; + + int ret = sscanf( + line, "%11[^,], %15[^,], %15[^,], %hu, %hu, %u, %u, %d, %d,%*s", + synack, srcip, dstip, &sport, &dport, &seq, &seq_ack, &cooldown, + &repeat); + + log_trace("forge-socket", + "%d '%s' sip: '%s', dip: '%s', sport: %d, dport: %d, seq: " + "%d, seq_ack: %d", + ret, synack, srcip, dstip, sport, dport, seq, seq_ack); + + if (ret == 9 && !repeat && strcmp(synack, "synack") == 0) { + st = malloc(sizeof(*st)); + st->conf = conf; + st->src_ip = inet_addr(srcip); + st->dst_ip = inet_addr(dstip); + st->sport = sport; + st->dport = dport; + st->seq = seq; + st->seq_ack = seq_ack; + + conf->current_running++; + grab_banner(st); + } + } +} + +int main(int argc, char *argv[]) { + struct event_base *base; + struct event * status_timer; + struct timeval status_timeout = {1, 0}; + int c; + struct option long_options[] = {{"concurrent", required_argument, 0, 'c'}, + {"read-timeout", required_argument, 0, 'r'}, + {"verbosity", required_argument, 0, 'v'}, + {"format", no_argument, 0, 'f'}, + {"data", required_argument, 0, 'd'}, + {0, 0, 0, 0}}; + + struct config conf; + int ret; + FILE * fp; + + log_init(stderr, LOG_INFO, 1, "forge-socket"); + + ret = + ulimit(4, 1000000); // Allow us to open 1 million fds (instead of 1024) + if (ret < 0) { + log_fatal("forge-socket", "cannot set ulimit"); + perror("ulimit"); + exit(1); + } + + base = event_base_new(); + conf.base = base; + + // buffer stdin as an event + conf.stdin_bev = bufferevent_socket_new(base, 0, BEV_OPT_DEFER_CALLBACKS); + bufferevent_setcb(conf.stdin_bev, stdin_readcb, NULL, stdin_eventcb, &conf); + bufferevent_enable(conf.stdin_bev, EV_READ); + + // Status timer + status_timer = evtimer_new(base, print_status, &conf); + evtimer_add(status_timer, &status_timeout); + + // Defaults + conf.max_concurrent = 1; + conf.current_running = 0; + memset(&conf.stats, 0, sizeof(conf.stats)); + conf.read_timeout = 4; + conf.stdin_closed = 0; + conf.format = FORMAT_BASE64; + conf.send_str = NULL; + + // Parse command line args + while (1) { + int option_index = 0; + c = getopt_long(argc, argv, "c:t:r:v:f:d:", long_options, + &option_index); + + if (c < 0) { + break; + } + + switch (c) { + case 'c': + conf.max_concurrent = atoi(optarg); + break; + case 'r': + conf.read_timeout = atoi(optarg); + break; + case 'v': + if (atoi(optarg) >= 0 && atoi(optarg) <= 5) { + log_init(stderr, atoi(optarg), 1, "forge-socket"); + } + break; + case 'f': + if (strcmp(optarg, "hex") == 0) { + conf.format = FORMAT_HEX; + } else if (strcmp(optarg, "base64") == 0) { + conf.format = FORMAT_BASE64; + } else if (strcmp(optarg, "ascii") == 0) { + conf.format = FORMAT_ASCII; + } else { + log_fatal( + "forge-socket", + "Unknown format '%s'; use 'hex', 'base64', or 'ascii'", + optarg); + } + break; + case 'd': + fp = fopen(optarg, "r"); + if (!fp) { + log_error("forge-socket", + "Could not open send data file '%s':", optarg); + perror("fopen"); + exit(-1); + } + fseek(fp, 0L, SEEK_END); + conf.send_str_size = ftell(fp); + fseek(fp, 0L, SEEK_SET); + // assert(conf.send_str_size < 10000); // jumbo frames? + conf.send_str = malloc(conf.send_str_size + 1); + if (!conf.send_str) { + log_fatal("forge-socket", "Could not malloc %d bytes", + conf.send_str_size + 1); + } + if (fread(conf.send_str, conf.send_str_size, 1, fp) != 1) { + log_fatal("forge-socket", + "Couldn't read from send data file '%s':", optarg); + } + conf.send_str[conf.send_str_size] = '\0'; + fclose(fp); + break; + case '?': + printf("Usage:\n"); + printf("\t%s [-c max_concurrency] [-r read_timeout] \n\t" + "[-v verbosity=0-5] [-d send_data_file] [-f " + "ascii|hex|base64]\n", + argv[0]); + exit(1); + default: + log_info("forge-socket", "hmmm.."); + break; + } + } + + log_info("forge-socket", "Using max_concurrency %d, %d s read timeout", + conf.max_concurrent, conf.read_timeout); + + event_base_dispatch(base); + + return 0; +} diff --git a/examples/udp-probes/123.hex b/examples/udp-probes/123.hex new file mode 100755 index 0000000..12613b1 --- /dev/null +++ b/examples/udp-probes/123.hex @@ -0,0 +1 @@ +e30004fa000100000001000000000000000000000000000000000000000000000000000000000000c54f234b71b152f3 \ No newline at end of file diff --git a/examples/udp-probes/137.hex b/examples/udp-probes/137.hex new file mode 100755 index 0000000..0ac1f8c --- /dev/null +++ b/examples/udp-probes/137.hex @@ -0,0 +1 @@ +80f00010000100000000000020434b4141414141414141414141414141414141414141414141414141414141410000210001 \ No newline at end of file diff --git a/examples/udp-probes/161.hex b/examples/udp-probes/161.hex new file mode 100755 index 0000000..00dd37f --- /dev/null +++ b/examples/udp-probes/161.hex @@ -0,0 +1 @@ +302602010104067075626c6963a1190204dc63c29a020100020100300b300906052b060102010500 \ No newline at end of file diff --git a/examples/udp-probes/1900.txt b/examples/udp-probes/1900.txt new file mode 100644 index 0000000..bcc633c --- /dev/null +++ b/examples/udp-probes/1900.txt @@ -0,0 +1,8 @@ +NOTIFY * HTTP/1.1 +Host:239.255.255.250:1900 +Cache-control:max-age=1800 +Location:http://[2402:f000:4:1:808::45]/description.xml +Nt:upnp:rootdevice +Nts:ssdp:alive +Usn:uuid:de5d6118-bfcb-918e-0000-00001eccef34::upnp:rootdevice + diff --git a/examples/udp-probes/520.hex b/examples/udp-probes/520.hex new file mode 100755 index 0000000..39090ad --- /dev/null +++ b/examples/udp-probes/520.hex @@ -0,0 +1 @@ +010100000000000000000000000000000000000000000010 \ No newline at end of file diff --git a/examples/udp-probes/53.hex b/examples/udp-probes/53.hex new file mode 100755 index 0000000..6bc37e6 --- /dev/null +++ b/examples/udp-probes/53.hex @@ -0,0 +1 @@ +53c0012000010000000000010377777705626169647503636f6d00000100010000291000000000000000 \ No newline at end of file diff --git a/examples/udp-probes/README b/examples/udp-probes/README new file mode 100644 index 0000000..99e461e --- /dev/null +++ b/examples/udp-probes/README @@ -0,0 +1,49 @@ + +UDP Data Probes +====== + +This directory contains a set of data files that can be used with the UDP probe module. + + +USING: +----- + +$ xmap -M udp -p 137 --probe-args=file:examples/udp-probes/netbios_137.pkt + + +PROBES: +----- + +citrix_1604.pkt This probe triggers a response from Citrix application discovery services on UDP port 1604 +db2disco_523.pkt This probe triggers a response from IBM DB2 discovery services on UDP port 523 +digi1_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (default magic) +digi2_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (devkit magic) +digi3_2362.pkt This probe triggers a response from Digi ADDP discovery services on UDP port 2362 (oem magic) +dns_53.pkt This probe queries for the DNS vendor and version using the BIND version TXT record over UDP port 53 +dns_53_queryAwww.google.it.pkt This probe queries for the domain www.google.it A record over UDP port 53 +dns_53_queryAwww.google.com.pkt This probe queries for the domain www.google.com A record over UDP port 53 +ipmi_623.pkt This probe triggers a Get Channel Authentication reply from IPMI endpoints on UDP port 623 +mdns_5353.pkt This probe triggers a response from mDNS/Avahi/Bonjour discovery services on UDP port 5353 +memcache_11211.pkt This probe triggers a response from memcached on UDP port 11211 (stats items). +mssql_1434.pkt This probe triggers a response from Microsoft SQL Server discovery services on UDP port 1434 +natpmp_5351.pkt This probe triggers a response from NATPMP-enabled devices on UDP port 5351 +netbios_137.pkt This probe triggers a status reply from NetBIOS services on UDP port 137 +ntp_123.pkt This probe triggers a response from NTP servies on UDP port 123 +ntp_123_monlist.pkt This probe triggers a response for command "monlist" from NTP servies on UDP port 123 +pca_nq_5632.pkt This probe triggers a response from PC Anywhere services on UDP port 5632 (network query) +pca_st_5632.pkt This probe triggers a response from PC Anywhere services on UDP port 5632 (status) +portmap_111.pkt This probe triggers a response from SunRPC portmapper services on UDP port 111 +ripv1_520.pkt This probe triggers a response from the RIPv1 enabled routers/devices on UDP port 520 +sentinel_5093.pkt This probe triggers a response from the Sentinel license manager service on UDP port 5093 +snmp1_161.pkt This probe queries for the system description field of SNMP v1 services using community string public over UDP port 161 +snmp2_161.pkt This probe queries for the system description field of SNMP v2 services using community string public over UDP port 161 +snmp3_161.pkt This probe triggers a response from SNMP v3 services on UDP port 161 +upnp_1900.pkt This probe triggers a response from UPnP SSDP services on UDP port 1900 +wdbrpc_17185.pkt This probe triggers a response from VxWorks WDBRPC services on UDP port 17185 +wsd_3702.pkt This probe triggers a response from WSD/DPWS services on UDP port 3702 +coap_5683.pkt This probe triggers a response from COAP services on UDP port 5683 + +NOTES: +----- + +Most of these probes return useful data in the response. Parsing this data requires capturing the raw output and decoding this using a protocol-specific dissector. In most cases, Wireshark is capable of decoding these replies. diff --git a/examples/udp-probes/bacnet_rpm_47808.pkt b/examples/udp-probes/bacnet_rpm_47808.pkt new file mode 100644 index 0000000000000000000000000000000000000000..041c9568a825a4d27b0d953c35fe256dbab8d2f0 GIT binary patch literal 37 scmZqVVo+sdVPa+E<6&Z8V36bF;gsRj;k4ki9GIDnfaddGBVh9fOF)%eSFkk=xG}#21 literal 0 HcmV?d00001 diff --git a/examples/udp-probes/digi1_2362.pkt b/examples/udp-probes/digi1_2362.pkt new file mode 100644 index 0000000000000000000000000000000000000000..74f57b88f68d8dd24a9972ffdb95e10e3e0d53d4 GIT binary patch literal 14 RcmZ?qboXRnWMKOb1^^=v2Oj_c literal 0 HcmV?d00001 diff --git a/examples/udp-probes/digi2_2362.pkt b/examples/udp-probes/digi2_2362.pkt new file mode 100644 index 0000000000000000000000000000000000000000..d962606efade4a7fe8103769fcb15c68a1a69a62 GIT binary patch literal 14 RcmZ<>^A2HPWMKOb1^^^t2Ri@& literal 0 HcmV?d00001 diff --git a/examples/udp-probes/digi3_2362.pkt b/examples/udp-probes/digi3_2362.pkt new file mode 100644 index 0000000000000000000000000000000000000000..ffba12522e33d978a04cb02e9f251f8326f381c0 GIT binary patch literal 14 RcmZ>9cL`u%WMKOb1^^=;2O$6e literal 0 HcmV?d00001 diff --git a/examples/udp-probes/dns_53.pkt b/examples/udp-probes/dns_53.pkt new file mode 100644 index 0000000000000000000000000000000000000000..616e17e8c6d512a676c5c60d14091fe3ab95e32f GIT binary patch literal 30 hcmXqc&&a?4L?FN(<{A|2>F>wlOa1OQoB1?>O; literal 0 HcmV?d00001 diff --git a/examples/udp-probes/dns_53_queryAwww.google.it.pkt b/examples/udp-probes/dns_53_queryAwww.google.it.pkt new file mode 100644 index 0000000000000000000000000000000000000000..99238c0f957efc324a7fae5af82f2cb3a527eddd GIT binary patch literal 31 gcmeZfWn^FgA`oCMFE3|H&(BZKNoC3`0SPk#07Rb!9{>OV literal 0 HcmV?d00001 diff --git a/examples/udp-probes/ipmi_623.pkt b/examples/udp-probes/ipmi_623.pkt new file mode 100644 index 0000000000000000000000000000000000000000..025d0fa01a82e5bc97fe5238be7ed732cf43fee4 GIT binary patch literal 23 YcmZQ$_|MJ&1DpyHCmIffO!q)L+@VaAVMC0B6Jn%>V!Z literal 0 HcmV?d00001 diff --git a/examples/udp-probes/ntp_123_monlist.pkt b/examples/udp-probes/ntp_123_monlist.pkt new file mode 100644 index 0000000000000000000000000000000000000000..ba2912d265d286d8348d046996d4eabeb08ec85b GIT binary patch literal 192 OcmWeJOD)i literal 0 HcmV?d00001 diff --git a/examples/udp-probes/pca_nq_5632.pkt b/examples/udp-probes/pca_nq_5632.pkt new file mode 100644 index 0000000..8d51173 --- /dev/null +++ b/examples/udp-probes/pca_nq_5632.pkt @@ -0,0 +1 @@ +NQ \ No newline at end of file diff --git a/examples/udp-probes/pca_st_5632.pkt b/examples/udp-probes/pca_st_5632.pkt new file mode 100644 index 0000000..86aa1fa --- /dev/null +++ b/examples/udp-probes/pca_st_5632.pkt @@ -0,0 +1 @@ +ST \ No newline at end of file diff --git a/examples/udp-probes/portmap_111.pkt b/examples/udp-probes/portmap_111.pkt new file mode 100644 index 0000000000000000000000000000000000000000..9143734872757d755349c0d832798124020f9226 GIT binary patch literal 40 bcmYc-;xcD|044^;wgo^Y6A-gNgb*YENW20_ literal 0 HcmV?d00001 diff --git a/examples/udp-probes/qotd_17.pkt b/examples/udp-probes/qotd_17.pkt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/udp-probes/qotd_17.pkt @@ -0,0 +1 @@ + diff --git a/examples/udp-probes/ripv1_520.pkt b/examples/udp-probes/ripv1_520.pkt new file mode 100644 index 0000000000000000000000000000000000000000..56ea97bb925dcf600a776a1deb157e0612499acf GIT binary patch literal 24 NcmZQ%WWWLh7ytlg022TJ literal 0 HcmV?d00001 diff --git a/examples/udp-probes/sentinel_5093.pkt b/examples/udp-probes/sentinel_5093.pkt new file mode 100644 index 0000000000000000000000000000000000000000..158b3a9dd9dcfc088b7fa278d91ac48eb8605ea3 GIT binary patch literal 6 KcmbR90LWy7VyG~L`u{&v2*|R5$}zzt0r6V})Bpeg literal 0 HcmV?d00001 diff --git a/examples/udp-probes/wsd_3702.pkt b/examples/udp-probes/wsd_3702.pkt new file mode 100644 index 0000000..87704a5 --- /dev/null +++ b/examples/udp-probes/wsd_3702.pkt @@ -0,0 +1,3 @@ + + +urn:schemas-xmlsoap-org:ws:2005:04:discoveryhttp://schemas.xmlsoap.org/ws/2005/04/discovery/Probeurn:uuid:ce04dad0-5d2c-4026-9146-1aabfc1e4111wsdp:Device diff --git a/examples/udp-probes/wsd_malformed_3702.pkt b/examples/udp-probes/wsd_malformed_3702.pkt new file mode 100644 index 0000000..9fd51b3 --- /dev/null +++ b/examples/udp-probes/wsd_malformed_3702.pkt @@ -0,0 +1 @@ +<:/> diff --git a/examples/udp-probes/xdmcp_177.pkt b/examples/udp-probes/xdmcp_177.pkt new file mode 100644 index 0000000000000000000000000000000000000000..96a017a5febf4f8c862591a68ac7841f3f00dc40 GIT binary patch literal 7 OcmZQzWME=oWB>pF7XSqS literal 0 HcmV?d00001 diff --git a/format.sh b/format.sh new file mode 100644 index 0000000..7380192 --- /dev/null +++ b/format.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +set -o pipefail + +MAJOR_REV=$((clang-format --version | awk '{print $3}' | cut -d '.' -f 1) || echo 0) +if [ $MAJOR_REV -lt 5 ]; then + echo "error: need at least clang-format version 5.x" + exit 1 +fi + +FORMAT_CMD="clang-format -i -style=file" + +# No files passed, format everything +if [ $# -eq 0 ]; then + echo "formatting all C code in src/ and lib/" + find ./src -type f -name '*.c' -exec $FORMAT_CMD {} \; + find ./src -type f -name '*.h' -exec $FORMAT_CMD {} \; + find ./lib -type f -name '*.c' -exec $FORMAT_CMD {} \; + find ./lib -type f -name '*.h' -exec $FORMAT_CMD {} \; + exit 0 +fi + +# File names passed, format only those files +$FORMAT_CMD $@ diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 0000000..54c797f --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,28 @@ +# sources file +set(LIB_SOURCES + blocklist.c + bloom.c + constraint.c + csv.c + gmp-ext.c + lockfd.c + logger.c + random.c + rijndael-alg-fst.c + util.c + xalloc.c + ) + +if (WITH_REDIS) + SET(LIB_SOURCES ${LIB_SOURCES} redis.c) +endif() + +# gen link target +add_library(xmaplib + STATIC ${LIB_SOURCES} + ) + +# specify the dir for gen target +target_include_directories(xmaplib + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} + ) diff --git a/lib/blocklist.c b/lib/blocklist.c new file mode 100644 index 0000000..3833324 --- /dev/null +++ b/lib/blocklist.c @@ -0,0 +1,485 @@ +/* + * Blocklist Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "blocklist.h" + +#include +#include +#include +#include +#include + +#include "constraint.h" +#include "gmp-ext.h" +#include "logger.h" +#include "xalloc.h" + +#define ADDR_DISALLOWED 0 +#define ADDR_ALLOWED 1 + +// scanning range on right, like 28 of 2001::/20-28 we just take bits from 0 to +// 28 for blocklisting as if a new IPvX is defined with max-length 28 new +// version: port range added we append the port bits to the end of the IPvX bits +// to gen random IPvX: ip + port + +static int IPVX_MAX_PREFIX_LEN = 128; // max IP bits +static int PORT_MAX_BITS = 16; // max port bits +static int IPVX_PORT_MAX_BITS = 144; // max IP + port bits + +static int IPV46_FLAG = 6; // IPv4/IPv6 flag +static int IP_MAX_PREFIX_LEN = 128; // IPv4/IPv6 max bits +static int IP_MAX_PREFIX_BYTES = 16; // IPv4/IPv6 max bits + +typedef struct bl_linked_list { + bl_cidr_node_t *first; + bl_cidr_node_t *last; + mpz_t len; +} bl_ll_t; + +static constraint_t *constraint = NULL; + +// keep track of the prefixes we've tried to BL/WL +// for logging purposes +static bl_ll_t *blocklisted_cidrs = NULL; +static bl_ll_t *allowlisted_cidrs = NULL; + +bl_cidr_node_t *get_blocklisted_cidrs(void) { return blocklisted_cidrs->first; } + +bl_cidr_node_t *get_allowlisted_cidrs(void) { return allowlisted_cidrs->first; } + +void blocklist_lookup_index_for_ipvx_port(mpz_t ipvx, const mpz_t index) { + constraint_lookup_index_for_ipvx_ui(ipvx, constraint, index, ADDR_ALLOWED); +} + +// check whether a single IP address is allowed to be scanned. +// appending port bits +int blocklist_is_allowed_ipvx(const mpz_t ipvx) { + mpz_t ipvx_p; + mpz_init(ipvx_p); + mpz_mul_2exp(ipvx_p, ipvx, PORT_MAX_BITS); + int ret = + constraint_lookup_ipvx_for_value_ui(constraint, ipvx_p) == ADDR_ALLOWED; + mpz_clear(ipvx_p); + + return ret; +} + +int blocklist_is_allowed_ip(const uint8_t *ip) { + mpz_t prefix; + mpz_init(prefix); + mpz_from_uint8s_bits(prefix, ip, IPVX_MAX_PREFIX_LEN); + int ret = blocklist_is_allowed_ipvx(prefix); + mpz_clear(prefix); + + return ret; +} + +static void bl_ll_add(bl_ll_t *l, const mpz_t addr, uint16_t p) { + assert(l); + bl_cidr_node_t *new = xmalloc(sizeof(bl_cidr_node_t)); + new->next = NULL; + mpz_init_set(new->ipvx_address, addr); + new->prefix_len = p; + + if (!l->first) { + l->first = new; + } else { + l->last->next = new; + } + l->last = new; + mpz_add_ui(l->len, l->len, 1); +} + +// appending port bits for constraint +// not for cidr list +static void _add_constraint(const mpz_t prefix, int prefix_len, int value) { + mpz_t ipvx_p; + mpz_init(ipvx_p); + mpz_mul_2exp(ipvx_p, prefix, PORT_MAX_BITS); + constraint_set_ui(constraint, ipvx_p, prefix_len, value); + mpz_clear(ipvx_p); + + if (value == ADDR_ALLOWED) { + bl_ll_add(allowlisted_cidrs, prefix, prefix_len); + } else if (value == ADDR_DISALLOWED) { + bl_ll_add(blocklisted_cidrs, prefix, prefix_len); + } else { + log_fatal("blocklist", "unknown type of blocklist operation specified"); + } +} + +// blocklist a CIDR network allocation +void blocklist_prefix(const mpz_t prefix, int prefix_len) { + _add_constraint(prefix, prefix_len, ADDR_DISALLOWED); +} + +// allowlist a CIDR network allocation +void allowlist_prefix(const mpz_t prefix, int prefix_len) { + _add_constraint(prefix, prefix_len, ADDR_ALLOWED); +} + +static int init_from_string(char *ip, int value, const char *name, + int ignore_invalid_hosts) { + char *dash = strchr(ip, '-'); + if (dash) { + *dash = '\0'; + log_debug("blocklist", "IPv%d address: %s", IPV46_FLAG, ip); + log_debug("blocklist", "IPv%d address: %s", IPV46_FLAG, dash + 1); + + uint8_t ip_start[IP_MAX_PREFIX_BYTES]; + uint8_t ip_end[IP_MAX_PREFIX_BYTES]; + inet_str2in(ip, ip_start, IPV46_FLAG); + inet_str2in(dash + 1, ip_end, IPV46_FLAG); + + mpz_t ip_start_m, ip_end_m; + mpz_init(ip_start_m); + mpz_init(ip_end_m); + mpz_from_uint8s(ip_start_m, ip_start, IP_MAX_PREFIX_BYTES); + mpz_from_uint8s(ip_end_m, ip_end, IP_MAX_PREFIX_BYTES); + + char ip_str[64]; + while (mpz_le(ip_start_m, ip_end_m)) { + mpz_to_uint8s(ip_start_m, ip_start, IP_MAX_PREFIX_BYTES); + inet_in2str(ip_start, ip_str, 64, IPV46_FLAG); + + int ret = + init_from_string(ip_str, value, name, ignore_invalid_hosts); + if (ret && !ignore_invalid_hosts) { + log_debug("blocklist", + "'%s' is not a valid IPv%d address or hostname", + IPV46_FLAG, ip_str); + return -1; + } + mpz_add_ui(ip_start_m, ip_start_m, 1); + } + + mpz_clear(ip_start_m); + mpz_clear(ip_end_m); + + return 0; + } else { + int prefix_len = IPVX_MAX_PREFIX_LEN; + char *slash = strchr(ip, '/'); + if (slash) { + *slash = '\0'; + char *len = slash + 1; + char *end; + errno = 0; + prefix_len = strtol(len, &end, 10); + if (end == len || errno != 0 || prefix_len < 0 || + prefix_len > IP_MAX_PREFIX_LEN) { + log_debug("blocklist", + "'%s' is not a valid cidr with length:%s", ip, len); + return -1; + } + } + + if (prefix_len > IPVX_MAX_PREFIX_LEN) { + log_debug( + "blocklist", + "no need to %s %s/%d for block-len:%d > max-scanning-len:%d", + name, ip, prefix_len, prefix_len, IPVX_MAX_PREFIX_LEN); + return 0; + } + + mpz_t addr; + mpz_init(addr); + struct addrinfo hint, *res; + memset(&hint, 0, sizeof(hint)); + int ret = -1; + + if (IPV46_FLAG == IPV6_FLAG) { + struct in6_addr ipv6; + if (inet_str2in(ip, &ipv6, IPV46_FLAG)) { // ipv6 + mpz_from_uint8s_bits(addr, (uint8_t *) &ipv6, + IPVX_MAX_PREFIX_LEN); + _add_constraint(addr, prefix_len, value); + log_debug("blocklist", "%sing: %s/%d", name, ip, prefix_len); + ret = 0; + goto cleanup; + } else { + goto pdns; + } + } else { + struct in_addr ipv4; + if (inet_str2in(ip, &ipv4, IPV46_FLAG)) { // ipv4 + mpz_from_uint8s_bits(addr, (uint8_t *) &ipv4, + IPVX_MAX_PREFIX_LEN); + _add_constraint(addr, prefix_len, value); + log_debug("blocklist", "%sing: %s/%d", name, ip, prefix_len); + ret = 0; + goto cleanup; + } else { + goto pdns; + } + } + + pdns: + hint.ai_protocol = IPPROTO_UDP; + if (getaddrinfo(ip, NULL, &hint, &res)) { + log_debug("blocklist", + "'%s' is not a valid IPv%d address or hostname", + IPV46_FLAG, ip); + goto cleanup; + } + + // Got some addrinfo, let's see what happens + char ip_str[64]; + for (struct addrinfo *aip = res; aip; aip = aip->ai_next) { + if (IPV46_FLAG == IPV6_FLAG) { + if (aip->ai_family != PF_INET6) continue; + struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) aip->ai_addr; + mpz_from_uint8s_bits(addr, (uint8_t *) &(sa6->sin6_addr), + IPVX_MAX_PREFIX_LEN); + inet_in2str((struct in6_addr *) &(sa6->sin6_addr), ip_str, 64, + IPV6_FLAG); + } else { + if (aip->ai_family != PF_INET) continue; + struct sockaddr_in *sa = (struct sockaddr_in *) aip->ai_addr; + mpz_from_uint8s_bits(addr, (uint8_t *) &(sa->sin_addr), + IPVX_MAX_PREFIX_LEN); + inet_in2str((struct in_addr *) &(sa->sin_addr), ip_str, 64, + IPV4_FLAG); + } + log_debug("blocklist", "%sing: %s(%s)/%d", name, ip, ip_str, + prefix_len); + ret = 0; + _add_constraint(addr, prefix_len, value); + } + + cleanup: + mpz_clear(addr); + + return ret; + } +} + +static int init_from_file(char *file, const char *name, int value, + int ignore_invalid_hosts) { + log_debug("blocklist", "%sing from file: %s", name, file); + + FILE *fp; + char line[1000]; + + fp = fopen(file, "r"); + if (fp == NULL) { + log_fatal(name, "unable to open %s file: %s: %s", name, file, + strerror(errno)); + } + + while (fgets(line, sizeof(line), fp) != NULL) { + char *comment = strchr(line, '#'); + if (comment) { + *comment = '\0'; + } + + // hostnames can be up to 255 bytes + char ip[256]; + if ((sscanf(line, "%256s", ip)) == EOF) { + continue; + } + if (init_from_string(ip, value, name, ignore_invalid_hosts)) { + if (!ignore_invalid_hosts) { + log_fatal(name, "unable to parse %s file: %s for IPv%d", name, + file, IPV46_FLAG); + } + } + } + fclose(fp); + + return 0; +} + +static void init_from_array(char **cidrs, size_t len, const char *name, + int value, int ignore_invalid_hosts) { + log_debug("blocklist", "%sing from input", name); + for (int i = 0; i < (int) len; i++) { + int ret = init_from_string(cidrs[i], value, name, ignore_invalid_hosts); + if (ret && !ignore_invalid_hosts) { + log_fatal("constraint", "Unable to init from CIDR list: %s", + cidrs[i]); + } + } +} + +// allowed: ip x port +void blocklist_count_allowed_ip_port(mpz_t count) { + assert(constraint); + constraint_count_ipvx_of_value_ui(count, constraint, ADDR_ALLOWED); +} + +// allowed: ip +void blocklist_count_allowed_ip(mpz_t count) { + assert(constraint); + constraint_count_ipvx_of_value_ui(count, constraint, ADDR_ALLOWED); + mpz_fdiv_q_2exp(count, count, PORT_MAX_BITS); +} + +// not allowed: ip x port +void blocklist_count_not_allowed_ip_port(mpz_t count) { + assert(constraint); + constraint_count_ipvx_of_value_ui(count, constraint, ADDR_DISALLOWED); +} + +// not allowed: ip +void blocklist_count_not_allowed_ip(mpz_t count) { + assert(constraint); + constraint_count_ipvx_of_value_ui(count, constraint, ADDR_DISALLOWED); + mpz_fdiv_q_2exp(count, count, PORT_MAX_BITS); +} + +uint32_t blocklist_ipvx_for_value(const mpz_t ipvx) { + assert(constraint); + return constraint_lookup_ipvx_for_value_ui(constraint, ipvx); +} + +int blocklist_init(char *allowlist_filename, char *blocklist_filename, + char **allowlist_entries, size_t allowlist_entries_len, + char **blocklist_entries, size_t blocklist_entries_len, + int ignore_invalid_hosts, size_t ipvx_max_len, + size_t port_max_len, size_t ipv46_flag) { + assert(!constraint); + if (port_max_len < 0 || port_max_len > 65535) + log_fatal("blocklist", "port bits number:%d > max bits number:%d", + port_max_len, PORT_MAX_BITS); + + IPVX_MAX_PREFIX_LEN = ipvx_max_len; + PORT_MAX_BITS = port_max_len; + IPV46_FLAG = ipv46_flag; + switch (IPV46_FLAG) { + case IPV6_FLAG: + IP_MAX_PREFIX_LEN = 128; + IP_MAX_PREFIX_BYTES = 16; + if (IPVX_MAX_PREFIX_LEN > IP_MAX_PREFIX_LEN) + log_fatal("blocklist", "blocklist-len:%d > IPv%d-max-len:%d", + IPVX_MAX_PREFIX_LEN, IPV46_FLAG, IP_MAX_PREFIX_LEN); + break; + case IPV4_FLAG: + IP_MAX_PREFIX_LEN = 32; + IP_MAX_PREFIX_BYTES = 4; + if (IPVX_MAX_PREFIX_LEN > IP_MAX_PREFIX_LEN) + log_fatal("blocklist", "blocklist-len:%d > IPv%d-max-len:%d", + IPVX_MAX_PREFIX_LEN, IPV46_FLAG, IP_MAX_PREFIX_LEN); + break; + default: + log_fatal("blocklist", "not supported IPv%d", IPV46_FLAG); + } + + IPVX_PORT_MAX_BITS = IPVX_MAX_PREFIX_LEN + PORT_MAX_BITS; + log_debug("blocklist", "IPVX_MAX_PREFIX_LEN=%d", IPVX_MAX_PREFIX_LEN); + log_debug("blocklist", "IPv%d_MAX_PREFIX_LEN=%d", IPV46_FLAG, + IP_MAX_PREFIX_LEN); + log_debug("blocklist", "PORT_MAX_BITS=%d", PORT_MAX_BITS); + log_debug("blocklist", "max blocklist len=%d", IPVX_PORT_MAX_BITS); + + blocklisted_cidrs = xcalloc(1, sizeof(bl_ll_t)); + mpz_init_set_ui(blocklisted_cidrs->len, 0); + allowlisted_cidrs = xcalloc(1, sizeof(bl_ll_t)); + mpz_init_set_ui(allowlisted_cidrs->len, 0); + + if (allowlist_filename && allowlist_entries) { + log_warn("allowlist", + "both a allowlist file and destination addresses were " + "specified. The union of these two sources will be utilized."); + } + + if (allowlist_filename || allowlist_entries_len > 0) { + // using a allowlist, so default to allowing nothing + constraint = constraint_init_ui(ADDR_DISALLOWED, IPVX_PORT_MAX_BITS); + log_debug("blocklist", "blocklisting: all /%d+%d(port)", + IPVX_MAX_PREFIX_LEN, PORT_MAX_BITS); + if (allowlist_filename) { + init_from_file(allowlist_filename, "allowlist", ADDR_ALLOWED, + ignore_invalid_hosts); + } + if (allowlist_entries) { + init_from_array(allowlist_entries, allowlist_entries_len, + "allowlist", ADDR_ALLOWED, ignore_invalid_hosts); + } + } else { + // no allowlist, so default to allowing everything + log_debug("blocklist", + "no allowlist file or allowlist entries provided, set to " + "allow all /%d+%d(port)", + IPVX_MAX_PREFIX_LEN, PORT_MAX_BITS); + constraint = constraint_init_ui(ADDR_ALLOWED, IPVX_PORT_MAX_BITS); + } + + if (blocklist_filename) { + init_from_file(blocklist_filename, "blocklist", ADDR_DISALLOWED, + ignore_invalid_hosts); + } + + if (blocklist_entries) { + init_from_array(blocklist_entries, blocklist_entries_len, "blocklist", + ADDR_DISALLOWED, ignore_invalid_hosts); + } + + if (IPV46_FLAG == IPV6_FLAG) + init_from_string(strdup("::"), ADDR_DISALLOWED, "blocklist", + ignore_invalid_hosts); + else + init_from_string(strdup("0.0.0.0"), ADDR_DISALLOWED, "blocklist", + ignore_invalid_hosts); + + constraint_paint_value_ui(constraint, ADDR_ALLOWED); + mpz_t allowed, total; + mpz_init_set_ui(allowed, 0); + mpz_init_set_ui(total, 1); + mpz_mul_2exp(total, total, IPVX_MAX_PREFIX_LEN); + blocklist_count_allowed_ip(allowed); // just the number of ip + mpf_t rate, allowed_f, total_f; + mpf_init(rate); + mpf_init(allowed_f); + mpf_init(total_f); + mpf_set_z(allowed_f, allowed); + mpf_set_z(total_f, total); + mpf_div(rate, allowed_f, total_f); + log_debug("blocklist", + "%s addresses (%0.2f%% of address space/0-%d: %s) can be scanned", + mpz_to_str10(allowed), mpf_get_d(rate) * 100, ipvx_max_len, + mpz_to_str10(total)); + + int ret = EXIT_SUCCESS; + if (mpz_eq_ui(allowed, 0)) { + log_error( + "blocklist", + "no addresses are eligible to be scanned in the current " + "configuration. This may be because the blocklist being used by " + "XMap (%s) prevents any addresses from receiving probe packets.", + blocklist_filename); + ret = EXIT_FAILURE; + } + + mpz_clear(allowed); + mpz_clear(total); + mpf_clear(rate); + mpf_clear(allowed_f); + mpf_clear(total_f); + + return ret; +} + +static void bl_free(bl_cidr_node_t *node) { + if (node == NULL) return; + mpz_clear(node->ipvx_address); + bl_free(node->next); + free(node); +} + +void blocklist_free() { + constraint_free(constraint); + bl_free(blocklisted_cidrs->first); + mpz_clear(blocklisted_cidrs->len); + bl_free(allowlisted_cidrs->first); + mpz_clear(allowlisted_cidrs->len); + log_debug("blocklist", "cleaning up"); +} diff --git a/lib/blocklist.h b/lib/blocklist.h new file mode 100644 index 0000000..4921a35 --- /dev/null +++ b/lib/blocklist.h @@ -0,0 +1,56 @@ +/* + * Blocklist Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_BLOCKLIST_H +#define XMAP_BLOCKLIST_H + +#include +#include + +typedef struct bl_cidr_node { + mpz_t ipvx_address; + int prefix_len; + struct bl_cidr_node *next; +} bl_cidr_node_t; + +int blocklist_init(char *allowlist_filename, char *blocklist_filename, + char **allowlist_entries, size_t allowlist_entries_len, + char **blocklist_entries, size_t blocklist_entries_len, + int ignore_invalid_hosts, size_t ipvx_max_len, + size_t port_max_len, size_t ipv46_flag); + +void blocklist_prefix(const mpz_t prefix, int prefix_len); + +void allowlist_prefix(const mpz_t prefix, int prefix_len); + +void blocklist_count_allowed_ip_port(mpz_t count); + +void blocklist_count_not_allowed_ip_port(mpz_t count); + +void blocklist_count_allowed_ip(mpz_t count); + +void blocklist_count_not_allowed_ip(mpz_t count); + +void blocklist_lookup_index_for_ipvx_port(mpz_t ipvx, const mpz_t index); + +int blocklist_is_allowed_ipvx(const mpz_t ipvx); + +int blocklist_is_allowed_ip(const uint8_t *ip); + +uint32_t blocklist_ipvx_for_value(const mpz_t ipvx); + +bl_cidr_node_t *get_blocklisted_cidrs(void); + +bl_cidr_node_t *get_allowlisted_cidrs(void); + +void blocklist_free(); + +#endif // XMAP_BLOCKLIST_H diff --git a/lib/bloom.c b/lib/bloom.c new file mode 100644 index 0000000..aafbd39 --- /dev/null +++ b/lib/bloom.c @@ -0,0 +1,554 @@ +/******************************************************************************* +*** +*** Author: Tyler Barrus +*** email: barrust@gmail.com +*** +*** Version: 1.8.0 +*** +*** License: MIT 2015 +*** +*** change: add arg: len at 09/08/2020 +*** int bloom_filter_add_string(BloomFilter *bf, const char *str, int len); +*******************************************************************************/ + +#include "bloom.h" +#include /* O_RDWR */ +#include /* pow, exp */ +#include /* printf */ +#include +#include /* strlen */ +#include /* mmap, mummap */ +#include /* fstat */ +#include /* */ +#include /* close */ + +#define CHECK_BIT_CHAR(c, k) (c & (1 << (k))) +#define CHECK_BIT(A, k) (CHECK_BIT_CHAR(A[((k) / 8)], ((k) % 8))) +// #define set_bit(A,k) (A[((k) / 8)] |= (1 << ((k) % 8))) +// #define clear_bit(A,k) (A[((k) / 8)] &= ~(1 << ((k) % 8))) + +#if defined(_OPENMP) +#define ATOMIC _Pragma("omp atomic") +#define CRITICAL _Pragma("omp critical (bloom_filter_critical)") +#else +#define ATOMIC +#define CRITICAL +#endif + +/* define some constant magic looking numbers */ +#define CHAR_LEN 8 +#define LOG_TWO_SQUARED 0.4804530139182 +#define LOG_TWO 0.6931471805599453 + +/******************************************************************************* +*** PRIVATE FUNCTIONS +*******************************************************************************/ +static uint64_t *__default_hash(int num_hashes, const char *str, int len); + +static uint64_t __fnv_1a(const char *key, int len); + +static void __calculate_optimal_hashes(BloomFilter *bf); + +static void __read_from_file(BloomFilter *bf, FILE *fp, short on_disk, + const char *filename); + +static void __write_to_file(BloomFilter *bf, FILE *fp, short on_disk); + +static int __sum_bits_set_char(char c); + +static int __check_if_union_or_intersection_ok(BloomFilter *res, + BloomFilter *bf1, + BloomFilter *bf2); + +int bloom_filter_init_alt(BloomFilter *bf, uint64_t estimated_elements, + float false_positive_rate, + BloomHashFunction hash_function) { + if (estimated_elements <= 0 || estimated_elements > UINT64_MAX || + false_positive_rate <= 0.0 || false_positive_rate >= 1.0) { + return BLOOM_FAILURE; + } + bf->estimated_elements = estimated_elements; + bf->false_positive_probability = false_positive_rate; + __calculate_optimal_hashes(bf); + bf->bloom = calloc(bf->bloom_length + 1, + sizeof(char)); // pad to ensure no running off the end + bf->elements_added = 0; + bloom_filter_set_hash_function(bf, hash_function); + bf->__is_on_disk = 0; // not on disk + return BLOOM_SUCCESS; +} + +int bloom_filter_init_on_disk_alt(BloomFilter *bf, uint64_t estimated_elements, + float false_positive_rate, + const char * filepath, + BloomHashFunction hash_function) { + if (estimated_elements <= 0 || estimated_elements > UINT64_MAX || + false_positive_rate <= 0.0 || false_positive_rate >= 1.0) { + return BLOOM_FAILURE; + } + bf->estimated_elements = estimated_elements; + bf->false_positive_probability = false_positive_rate; + __calculate_optimal_hashes(bf); + bf->elements_added = 0; + FILE *fp; + fp = fopen(filepath, "w+b"); + if (fp == NULL) { + fprintf(stderr, "Can't open file %s!\n", filepath); + return BLOOM_FAILURE; + } + __write_to_file(bf, fp, 1); + fclose(fp); + // slightly ineffecient to redo some of the calculations... + return bloom_filter_import_on_disk_alt(bf, filepath, hash_function); +} + +void bloom_filter_set_hash_function(BloomFilter * bf, + BloomHashFunction hash_function) { + bf->hash_function = + (hash_function == NULL) ? __default_hash : hash_function; +} + +int bloom_filter_destroy(BloomFilter *bf) { + if (bf->__is_on_disk == 0) { + free(bf->bloom); + } else { + fclose(bf->filepointer); + munmap(bf->bloom, bf->__filesize); + } + bf->bloom = NULL; + bf->filepointer = NULL; + bf->elements_added = 0; + bf->estimated_elements = 0; + bf->false_positive_probability = 0; + bf->number_hashes = 0; + bf->number_bits = 0; + bf->hash_function = NULL; + bf->__is_on_disk = 0; + bf->__filesize = 0; + return BLOOM_SUCCESS; +} + +int bloom_filter_clear(BloomFilter *bf) { + long i; + for (i = 0; i < bf->bloom_length; ++i) { + bf->bloom[i] = 0; + } + bf->elements_added = 0; + return BLOOM_SUCCESS; +} + +void bloom_filter_stats(BloomFilter *bf) { + const char *is_on_disk = (bf->__is_on_disk == 0 ? "no" : "yes"); + uint64_t size_on_disk = bloom_filter_export_size(bf); + + printf("BloomFilter\n\ + bits: %" PRIu64 "\n\ + estimated elements: %" PRIu64 "\n\ + number hashes: %d\n\ + max false positive rate: %f\n\ + bloom length (8 bits): %ld\n\ + elements added: %" PRIu64 "\n\ + estimated elements added: %" PRIu64 "\n\ + current false positive rate: %f\n\ + export size (bytes): %" PRIu64 "\n\ + number bits set: %" PRIu64 "\n\ + is on disk: %s\n", + bf->number_bits, bf->estimated_elements, bf->number_hashes, + bf->false_positive_probability, bf->bloom_length, bf->elements_added, + bloom_filter_estimate_elements(bf), + bloom_filter_current_false_positive_rate(bf), size_on_disk, + bloom_filter_count_set_bits(bf), is_on_disk); +} + +int bloom_filter_add_string(BloomFilter *bf, const char *str, int len) { + uint64_t *hashes = + bloom_filter_calculate_hashes(bf, str, bf->number_hashes, len); + int res = bloom_filter_add_string_alt(bf, hashes, bf->number_hashes); + free(hashes); + return res; +} + +int bloom_filter_check_string(BloomFilter *bf, const char *str, int len) { + uint64_t *hashes = + bloom_filter_calculate_hashes(bf, str, bf->number_hashes, len); + int res = bloom_filter_check_string_alt(bf, hashes, bf->number_hashes); + free(hashes); + return res; +} + +uint64_t *bloom_filter_calculate_hashes(BloomFilter *bf, const char *str, + unsigned int number_hashes, int len) { + return bf->hash_function(number_hashes, str, len); +} + +/* Add a string to a bloom filter using the defined hashes */ +int bloom_filter_add_string_alt(BloomFilter *bf, uint64_t *hashes, + unsigned int number_hashes_passed) { + if (number_hashes_passed < bf->number_hashes) { + fprintf(stderr, + "Error: not enough hashes passed in to correctly check!\n"); + return BLOOM_FAILURE; + } + + unsigned int i; + for (i = 0; i < bf->number_hashes; ++i) { + ATOMIC + bf->bloom[(hashes[i] % bf->number_bits) / 8] |= + (1u << ((hashes[i] % bf->number_bits) % 8)); // set the bit + } + + ATOMIC + ++bf->elements_added; + if (bf->__is_on_disk == 1) { // only do this if it is on disk! + int offset = sizeof(uint64_t) + sizeof(float); + CRITICAL { + fseek(bf->filepointer, offset * -1, SEEK_END); + fwrite(&bf->elements_added, sizeof(uint64_t), 1, bf->filepointer); + } + } + return BLOOM_SUCCESS; +} + +/* Check if a string is in the bloom filter using the passed hashes */ +int bloom_filter_check_string_alt(BloomFilter *bf, uint64_t *hashes, + unsigned int number_hashes_passed) { + if (number_hashes_passed < bf->number_hashes) { + fprintf(stderr, + "Error: not enough hashes passed in to correctly check!\n"); + return BLOOM_FAILURE; + } + + unsigned int i; + int r = BLOOM_SUCCESS; + for (i = 0; i < bf->number_hashes; ++i) { + int tmp_check = CHECK_BIT(bf->bloom, (hashes[i] % bf->number_bits)); + if (tmp_check == 0) { + r = BLOOM_FAILURE; + break; // no need to continue checking + } + } + return r; +} + +float bloom_filter_current_false_positive_rate(BloomFilter *bf) { + int num = (bf->number_hashes * -1 * bf->elements_added); + double d = num / (float) bf->number_bits; + double e = exp(d); + return pow((1 - e), bf->number_hashes); +} + +int bloom_filter_export(BloomFilter *bf, const char *filepath) { + // if the bloom is initialized on disk, no need to export it + if (bf->__is_on_disk == 1) { + return BLOOM_SUCCESS; + } + FILE *fp; + fp = fopen(filepath, "w+b"); + if (fp == NULL) { + fprintf(stderr, "Can't open file %s!\n", filepath); + return BLOOM_FAILURE; + } + __write_to_file(bf, fp, 0); + fclose(fp); + return BLOOM_SUCCESS; +} + +int bloom_filter_import_alt(BloomFilter *bf, const char *filepath, + BloomHashFunction hash_function) { + FILE *fp; + fp = fopen(filepath, "r+b"); + if (fp == NULL) { + fprintf(stderr, "Can't open file %s!\n", filepath); + return BLOOM_FAILURE; + } + __read_from_file(bf, fp, 0, NULL); + fclose(fp); + bloom_filter_set_hash_function(bf, hash_function); + bf->__is_on_disk = 0; // not on disk + return BLOOM_SUCCESS; +} + +int bloom_filter_import_on_disk_alt(BloomFilter *bf, const char *filepath, + BloomHashFunction hash_function) { + bf->filepointer = fopen(filepath, "r+b"); + if (bf->filepointer == NULL) { + fprintf(stderr, "Can't open file %s!\n", filepath); + return BLOOM_FAILURE; + } + __read_from_file(bf, bf->filepointer, 1, filepath); + // don't close the file pointer here... + bloom_filter_set_hash_function(bf, hash_function); + bf->__is_on_disk = 1; // on disk + return BLOOM_SUCCESS; +} + +char *bloom_filter_export_hex_string(BloomFilter *bf) { + uint64_t i, + bytes = sizeof(uint64_t) * 2 + sizeof(float) + (bf->bloom_length); + char *hex = malloc((bytes * 2 + 1) * sizeof(char)); + for (i = 0; i < (uint64_t) bf->bloom_length; ++i) { + sprintf(hex + (i * 2), "%02x", + bf->bloom[i]); // not the fastest way, but works + } + i = bf->bloom_length * 2; + sprintf(hex + i, "%016" PRIx64 "", bf->estimated_elements); + i += 16; // 8 bytes * 2 for hex + sprintf(hex + i, "%016" PRIx64 "", bf->elements_added); + + unsigned int ui; + memcpy(&ui, &bf->false_positive_probability, sizeof(ui)); + i += 16; // 8 bytes * 2 for hex + sprintf(hex + i, "%08x", ui); + return hex; +} + +int bloom_filter_import_hex_string_alt(BloomFilter *bf, const char *hex, + BloomHashFunction hash_function) { + uint64_t len = strlen(hex); + if (len % 2 != 0) { + fprintf(stderr, "Unable to parse; exiting\n"); + return BLOOM_FAILURE; + } + char fpr[9] = {0}; + char est_els[17] = {0}; + char ins_els[17] = {0}; + memcpy(fpr, hex + (len - 8), 8); + memcpy(ins_els, hex + (len - 24), 16); + memcpy(est_els, hex + (len - 40), 16); + uint32_t t_fpr; + + bf->estimated_elements = strtoull(est_els, NULL, 16); + bf->elements_added = strtoull(ins_els, NULL, 16); + sscanf(fpr, "%x", &t_fpr); + float f; + memcpy(&f, &t_fpr, sizeof(float)); + bf->false_positive_probability = f; + bloom_filter_set_hash_function(bf, hash_function); + + __calculate_optimal_hashes(bf); + bf->bloom = calloc(bf->bloom_length + 1, sizeof(char)); // pad + bf->__is_on_disk = 0; // not on disk + + uint64_t i; + for (i = 0; i < (uint64_t) bf->bloom_length; ++i) { + sscanf(hex + (i * 2), "%2hhx", &bf->bloom[i]); + } + return BLOOM_SUCCESS; +} + +uint64_t bloom_filter_export_size(BloomFilter *bf) { + return (uint64_t)(bf->bloom_length * sizeof(unsigned char)) + + (2 * sizeof(uint64_t)) + sizeof(float); +} + +uint64_t bloom_filter_count_set_bits(BloomFilter *bf) { + uint64_t i, res = 0; + for (i = 0; i < (uint64_t) bf->bloom_length; ++i) { + res += __sum_bits_set_char(bf->bloom[i]); + } + return res; +} + +uint64_t bloom_filter_estimate_elements(BloomFilter *bf) { + return bloom_filter_estimate_elements_by_values( + bf->number_bits, bloom_filter_count_set_bits(bf), bf->number_hashes); +} + +uint64_t bloom_filter_estimate_elements_by_values(uint64_t m, uint64_t X, + int k) { + /* m = number bits; X = count of flipped bits; k = number hashes */ + double log_n = log(1 - ((double) X / (double) m)); + return (uint64_t) - (((double) m / k) * log_n); +} + +int bloom_filter_union(BloomFilter *res, BloomFilter *bf1, BloomFilter *bf2) { + // Ensure the bloom filters can be unioned + if (__check_if_union_or_intersection_ok(res, bf1, bf2) == BLOOM_FAILURE) { + return BLOOM_FAILURE; + } + uint64_t i; + for (i = 0; i < (uint64_t) bf1->bloom_length; ++i) { + res->bloom[i] = bf1->bloom[i] | bf2->bloom[i]; + } + bloom_filter_set_elements_to_estimated(res); + return BLOOM_SUCCESS; +} + +uint64_t bloom_filter_count_union_bits_set(BloomFilter *bf1, BloomFilter *bf2) { + // Ensure the bloom filters can be unioned + if (__check_if_union_or_intersection_ok(bf1, bf1, bf2) == + BLOOM_FAILURE) { // use bf1 as res + return BLOOM_FAILURE; + } + uint64_t i, res = 0; + for (i = 0; i < (uint64_t) bf1->bloom_length; ++i) { + res += __sum_bits_set_char(bf1->bloom[i] | bf2->bloom[i]); + } + return res; +} + +int bloom_filter_intersect(BloomFilter *res, BloomFilter *bf1, + BloomFilter *bf2) { + // Ensure the bloom filters can be used in an intersection + if (__check_if_union_or_intersection_ok(res, bf1, bf2) == BLOOM_FAILURE) { + return BLOOM_FAILURE; + } + uint64_t i; + for (i = 0; i < (uint64_t) bf1->bloom_length; ++i) { + res->bloom[i] = bf1->bloom[i] & bf2->bloom[i]; + } + bloom_filter_set_elements_to_estimated(res); + return BLOOM_SUCCESS; +} + +void bloom_filter_set_elements_to_estimated(BloomFilter *bf) { + bf->elements_added = bloom_filter_estimate_elements(bf); +} + +uint64_t bloom_filter_count_intersection_bits_set(BloomFilter *bf1, + BloomFilter *bf2) { + // Ensure the bloom filters can be used in an intersection + if (__check_if_union_or_intersection_ok(bf1, bf1, bf2) == + BLOOM_FAILURE) { // use bf1 as res + return BLOOM_FAILURE; + } + uint64_t i, res = 0; + for (i = 0; i < (uint64_t) bf1->bloom_length; ++i) { + res += __sum_bits_set_char(bf1->bloom[i] & bf2->bloom[i]); + } + return res; +} + +float bloom_filter_jaccard_index(BloomFilter *bf1, BloomFilter *bf2) { + // Ensure the bloom filters can be used in an intersection and union + if (__check_if_union_or_intersection_ok(bf1, bf1, bf2) == + BLOOM_FAILURE) { // use bf1 as res + return (float) BLOOM_FAILURE; + } + float set_union_bits = (float) bloom_filter_count_union_bits_set(bf1, bf2); + if (set_union_bits == 0.0) { // check for divide by 0 error + return (float) 1.0; // they must be both empty for this to occur and are + // therefore the same + } + return (float) bloom_filter_count_intersection_bits_set(bf1, bf2) / + set_union_bits; +} + +/******************************************************************************* + * PRIVATE FUNCTIONS + *******************************************************************************/ +static void __calculate_optimal_hashes(BloomFilter *bf) { + // calc optimized values + long n = bf->estimated_elements; + float p = bf->false_positive_probability; + uint64_t m = ceil((-n * log(p)) / LOG_TWO_SQUARED); // AKA pow(log(2), 2); + unsigned int k = round(LOG_TWO * m / n); // AKA log(2.0); + // set paramenters + bf->number_hashes = k; // should check to make sure it is at least 1... + bf->number_bits = m; + long num_pos = ceil(m / (CHAR_LEN * 1.0)); + bf->bloom_length = num_pos; +} + +static int __sum_bits_set_char(char c) { + int j, res = 0; + for (j = 0; j < CHAR_LEN; ++j) { + res += (CHECK_BIT_CHAR(c, j) != 0) ? 1 : 0; + } + return res; +} + +static int __check_if_union_or_intersection_ok(BloomFilter *res, + BloomFilter *bf1, + BloomFilter *bf2) { + if (res->number_hashes != bf1->number_hashes || + bf1->number_hashes != bf2->number_hashes) { + return BLOOM_FAILURE; + } else if (res->number_bits != bf1->number_bits || + bf1->number_bits != bf2->number_bits) { + return BLOOM_FAILURE; + } else if (res->hash_function != bf1->hash_function || + bf1->hash_function != bf2->hash_function) { + return BLOOM_FAILURE; + } + return BLOOM_SUCCESS; +} + +/* NOTE: this assumes that the file handler is open and ready to use */ +static void __write_to_file(BloomFilter *bf, FILE *fp, short on_disk) { + if (on_disk == 0) { + fwrite(bf->bloom, bf->bloom_length, 1, fp); + } else { + // will need to write out everything by hand + uint64_t i; + for (i = 0; i < (uint64_t) bf->bloom_length; ++i) { + fputc(0, fp); + } + } + fwrite(&bf->estimated_elements, sizeof(uint64_t), 1, fp); + fwrite(&bf->elements_added, sizeof(uint64_t), 1, fp); + fwrite(&bf->false_positive_probability, sizeof(float), 1, fp); +} + +/* NOTE: this assumes that the file handler is open and ready to use */ +static void __read_from_file(BloomFilter *bf, FILE *fp, short on_disk, + const char *filename) { + int offset = sizeof(uint64_t) * 2 + sizeof(float); + fseek(fp, offset * -1, SEEK_END); + size_t read; + read = fread(&bf->estimated_elements, sizeof(uint64_t), 1, fp); + read = fread(&bf->elements_added, sizeof(uint64_t), 1, fp); + read = fread(&bf->false_positive_probability, sizeof(float), 1, fp); + __calculate_optimal_hashes(bf); + rewind(fp); + if (on_disk == 0) { + bf->bloom = calloc(bf->bloom_length + 1, sizeof(char)); + read = fread(bf->bloom, sizeof(char), bf->bloom_length, fp); + if (read != (uint64_t) bf->bloom_length) { + perror("__read_from_file: "); + exit(1); + } + } else { + struct stat buf; + int fd = open(filename, O_RDWR); + if (fd < 0) { + perror("open: "); + exit(1); + } + fstat(fd, &buf); + bf->__filesize = buf.st_size; + bf->bloom = mmap((caddr_t) 0, bf->__filesize, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (bf->bloom == (unsigned char *) -1) { + perror("mmap: "); + exit(1); + } + // close the file descriptor + close(fd); + } +} + +/* NOTE: The caller will free the results */ +static uint64_t *__default_hash(int num_hashes, const char *str, int len) { + uint64_t *results = calloc(num_hashes, sizeof(uint64_t)); + int i; + char key[32] = { + 0}; // largest value is 7FFF,FFFF,FFFF,FFFF TODO the real str len + results[0] = __fnv_1a(str, len); + for (i = 1; i < num_hashes; ++i) { + sprintf(key, "%" PRIx64 "", results[i - 1]); + results[i] = __fnv_1a(key, len); + } + return results; +} + +static uint64_t __fnv_1a(const char *key, int len) { + // FNV-1a hash (http://www.isthe.com/chongo/tech/comp/fnv/) + int i; + uint64_t h = 14695981039346656073ULL; // FNV_OFFSET 64 bit + for (i = 0; i < len; ++i) { + h = h ^ (unsigned char) key[i]; + h = h * 1099511628211ULL; // FNV_PRIME 64 bit + } + return h; +} diff --git a/lib/bloom.h b/lib/bloom.h new file mode 100644 index 0000000..db29ab8 --- /dev/null +++ b/lib/bloom.h @@ -0,0 +1,212 @@ +/******************************************************************************* +*** +*** Author: Tyler Barrus +*** email: barrust@gmail.com +*** +*** Version: 1.8.0 +*** Purpose: Simple, yet effective, bloom filter implementation +*** +*** License: MIT 2015 +*** +*** URL: https://github.com/barrust/bloom +*** +*** change: add arg: len at 09/08/2020 +*** int bloom_filter_add_string(BloomFilter *bf, const char *str, int len); +*******************************************************************************/ +#ifndef BARRUST_BLOOM_FILTER_H__ +#define BARRUST_BLOOM_FILTER_H__ + +#include /* PRIu64 */ +#include + +/* https://gcc.gnu.org/onlinedocs/gcc/Alternate-Keywords.html#Alternate-Keywords + */ +#ifndef __GNUC__ +#define __inline__ inline +#endif + +#ifdef __APPLE__ +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#define BLOOMFILTER_VERSION "1.8.0" +#define BLOOMFILTER_MAJOR 1 +#define BLOOMFILTER_MINOR 8 +#define BLOOMFILTER_REVISION 0 + +#define BLOOM_SUCCESS 0 +#define BLOOM_FAILURE -1 + +#define bloom_filter_get_version() (BLOOMFILTER_VERSION) + +typedef uint64_t *(*BloomHashFunction)(int num_hashes, const char *str, + int len); + +typedef struct bloom_filter { + /* bloom parameters */ + uint64_t estimated_elements; + float false_positive_probability; + unsigned int number_hashes; + uint64_t number_bits; + /* bloom filter */ + unsigned char * bloom; + long bloom_length; + uint64_t elements_added; + BloomHashFunction hash_function; + /* on disk handeling */ + short __is_on_disk; + FILE * filepointer; + uint64_t __filesize; +} BloomFilter; + +/* Initialize a standard bloom filter in memory; this will provide 'optimal' + size and hash numbers. Estimated elements is 0 < x <= UINT64_MAX. False + positive rate is 0.0 < x < 1.0 */ +int bloom_filter_init_alt(BloomFilter *bf, uint64_t estimated_elements, + float false_positive_rate, + BloomHashFunction hash_function); + +__inline__ static int bloom_filter_init(BloomFilter *bf, + uint64_t estimated_elements, + float false_positive_rate) { + return bloom_filter_init_alt(bf, estimated_elements, false_positive_rate, + NULL); +} + +/* Initialize a bloom filter directly into file; useful if the bloom filter is + * larger than available RAM */ +int bloom_filter_init_on_disk_alt(BloomFilter *bf, uint64_t estimated_elements, + float false_positive_rate, + const char * filepath, + BloomHashFunction hash_function); + +__inline__ static int bloom_filter_init_on_disk(BloomFilter *bf, + uint64_t estimated_elements, + float false_positive_rate, + const char *filepath) { + return bloom_filter_init_on_disk_alt(bf, estimated_elements, + false_positive_rate, filepath, NULL); +} + +/* Import a previously exported bloom filter from a file into memory */ +int bloom_filter_import_alt(BloomFilter *bf, const char *filepath, + BloomHashFunction hash_function); + +__inline__ static int bloom_filter_import(BloomFilter *bf, + const char * filepath) { + return bloom_filter_import_alt(bf, filepath, NULL); +} + +/* Import a previously exported bloom filter from a file but do not pull the + full bloom into memory. This is allows for the speed / storage trade off of + not needing to put the full bloom filter into RAM. */ +int bloom_filter_import_on_disk_alt(BloomFilter *bf, const char *filepath, + BloomHashFunction hash_function); + +__inline__ static int bloom_filter_import_on_disk(BloomFilter *bf, + const char * filepath) { + return bloom_filter_import_on_disk_alt(bf, filepath, NULL); +} + +/* Export the current bloom filter to file */ +int bloom_filter_export(BloomFilter *bf, const char *filepath); + +/* Export and import as a hex string; not space effecient but allows for + storing multiple blooms in a single file or in a database, etc. NOTE: It is + up to the caller to free the allocated memory */ +char *bloom_filter_export_hex_string(BloomFilter *bf); + +int bloom_filter_import_hex_string_alt(BloomFilter *bf, const char *hex, + BloomHashFunction hash_function); + +__inline__ static int bloom_filter_import_hex_string(BloomFilter *bf, + char * hex) { + return bloom_filter_import_hex_string_alt(bf, hex, NULL); +} + +/* Set or change the hashing function */ +void bloom_filter_set_hash_function(BloomFilter * bf, + BloomHashFunction hash_function); + +/* Print out statistics about the bloom filter */ +void bloom_filter_stats(BloomFilter *bf); + +/* Release all memory used by the bloom filter */ +int bloom_filter_destroy(BloomFilter *bf); + +/* reset filter to unused state */ +int bloom_filter_clear(BloomFilter *bf); + +/* Add a string (or element) to the bloom filter */ +int bloom_filter_add_string(BloomFilter *bf, const char *str, int len); + +/* Add a string to a bloom filter using the defined hashes */ +int bloom_filter_add_string_alt(BloomFilter *bf, uint64_t *hashes, + unsigned int number_hashes_passed); + +/* Check to see if a string (or element) is or is not in the bloom filter */ +int bloom_filter_check_string(BloomFilter *bf, const char *str, int len); + +/* Check if a string is in the bloom filter using the passed hashes */ +int bloom_filter_check_string_alt(BloomFilter *bf, uint64_t *hashes, + unsigned int number_hashes_passed); + +/* Calculates the current false positive rate based on the number of inserted + * elements */ +float bloom_filter_current_false_positive_rate(BloomFilter *bf); + +/* Count the number of bits set to 1 */ +uint64_t bloom_filter_count_set_bits(BloomFilter *bf); + +/* Estimate the number of unique elements in a Bloom Filter instead of using + the overall count + https://en.wikipedia.org/wiki/Bloom_filter#Approximating_the_number_of_items_in_a_Bloom_filter + m = bits in Bloom filter + k = number hashes + X = count of flipped bits in filter */ +uint64_t bloom_filter_estimate_elements(BloomFilter *bf); + +uint64_t bloom_filter_estimate_elements_by_values(uint64_t m, uint64_t X, + int k); + +/* Wrapper to set the inserted elements count to the estimated elements + * calculation */ +void bloom_filter_set_elements_to_estimated(BloomFilter *bf); + +/* Generate the desired number of hashes for the provided string + NOTE: It is up to the caller to free the allocated memory */ +uint64_t *bloom_filter_calculate_hashes(BloomFilter *bf, const char *str, + unsigned int number_hashes, int len); + +/* Calculate the size the bloom filter will take on disk when exported in bytes + */ +uint64_t bloom_filter_export_size(BloomFilter *bf); + +/******************************************************************************* + Merging, Intersection, and Jaccard Index Functions + NOTE: Requires that the bloom filters be of the same type: hash, estimated + elements, etc. +*******************************************************************************/ + +/* Merge Bloom Filters - inserts information into res */ +int bloom_filter_union(BloomFilter *res, BloomFilter *bf1, BloomFilter *bf2); + +uint64_t bloom_filter_count_union_bits_set(BloomFilter *bf1, BloomFilter *bf2); + +/* Find the intersection of Bloom Filters - insert into res with the + intersection The number of inserted elements is updated to the estimated + elements calculation */ +int bloom_filter_intersect(BloomFilter *res, BloomFilter *bf1, + BloomFilter *bf2); + +uint64_t bloom_filter_count_intersection_bits_set(BloomFilter *bf1, + BloomFilter *bf2); + +/* Calculate the Jacccard Index of the Bloom Filters + NOTE: The closer to 1 the index, the closer in bloom filters. If it is 1, + then the Bloom Filters contain the same elements, 0.5 would mean about 1/2 + the same elements are in common. 0 would mean the Bloom Filters are + completely different. */ +float bloom_filter_jaccard_index(BloomFilter *bf1, BloomFilter *bf2); + +#endif /* END BLOOM FILTER HEADER */ diff --git a/lib/constraint.c b/lib/constraint.c new file mode 100644 index 0000000..3e4e3e7 --- /dev/null +++ b/lib/constraint.c @@ -0,0 +1,601 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "constraint.h" + +#include +#include +#include + +#include "gmp-ext.h" +#include "logger.h" +#include "xalloc.h" + +// +// Efficient address-space constraints (AH 7/2013) +// +// This module uses a tree-based representation to efficiently manipulate and +// query constraints on the address space to be scanned.It provides a value for +// every IP(v6) address, and these values are applied by setting them for +// network prefixes. Order matters: setting a value replaces any existing value +// for that prefix or subsets of it. We use this to implement network whitelist +// and blacklist. +// +// Think of setting values in this structure like painting subnets with +// different colors. We can paint subnets black to exclude them and white to +// allow them. Only the top color shows. This makes for potentially very +// powerful constraint specifications. +// +// Internally, this is implemented using a binary tree, where each node +// corresponds to a network prefix. (E.g., the root is ::0.0.0.0/0(::/0), and +// its children, if present, are ::0.0.0.0/1(::/1) and +// 8000::0.0.0.0/1(8000::/1).) Each leaf of the tree stores the value that +// applies to every address within the leaf's portion of the prefix space. +// +// As an optimization, after all values are set, we look up the value or subtree +// for every /20 prefix and cache them as an array. +// + +// As an optimization, we precompute lookups for every prefix of this length. +// Mem at least is 2^len * 16B(for a mpz_t). + +// As an extension, we build a constraint accepting any MAX IP length. Further, +// the IPvX below could consist of a max IP length and max port length for +// genning random + +// IPvX mask +static int IPVX_MAX_PREFIX_LEN = 0; +static int RADIX_LENGTH = 20; +static int IPVX_MASK_FLAG = 0; + +static mpz_t IPVX_HALF_INDEX; +static mpz_t IPVX_MAX_INDEX; +static mpz_t RADIX_SIZE; + +typedef struct node { + struct node *l; + struct node *r; + mpz_t value; // value implying white, black or other + mpz_t count; // number of being painted +} node_t; + +struct _constraint { + node_t *root; // root node of the tree + mpz_t * radix; // array of prefixes (/RADIX_LENGTH) that + // are painted paint_value + size_t radix_len; // number of prefixes in radix array + int painted; // have we precomputed counts for each node? + mpz_t paint_value; // value for which we precomputed counts +}; + +// Tree operations respect the invariant that every node that isn't a +// leaf has exactly two children. +#define IS_LEAF(node) ((node)->l == NULL) + +// Allocate a new leaf with the given value +static node_t *_create_leaf(const mpz_t value) { + node_t *node = xmalloc(sizeof(node_t)); + node->l = NULL; + node->r = NULL; + mpz_init_set(node->value, value); + + return node; +} + +// calculate the initial IPvX mask +static void cal_ipvx_mask(size_t ip_max_len) { + assert(0 < ip_max_len && ip_max_len <= 144); + + if (!IPVX_MASK_FLAG) { + log_debug("constraint", "calculating mask for IPvX, max-len=%d", + ip_max_len); + IPVX_MAX_PREFIX_LEN = ip_max_len; + if (IPVX_MAX_PREFIX_LEN < RADIX_LENGTH) + RADIX_LENGTH = IPVX_MAX_PREFIX_LEN; + + mpz_t two; + mpz_init_set_ui(two, 2); + mpz_init(IPVX_HALF_INDEX); + mpz_pow_ui(IPVX_HALF_INDEX, two, IPVX_MAX_PREFIX_LEN - 1); + mpz_init(IPVX_MAX_INDEX); + mpz_pow_ui(IPVX_MAX_INDEX, two, IPVX_MAX_PREFIX_LEN); + mpz_init(RADIX_SIZE); + mpz_pow_ui(RADIX_SIZE, two, IPVX_MAX_PREFIX_LEN - RADIX_LENGTH); + mpz_clear(two); + IPVX_MASK_FLAG = 1; + + log_debug("constraint", "IPvX_MAX_PREFIX_LEN: %d", IPVX_MAX_PREFIX_LEN); + log_debug("constraint", "IPvX_HALF_INDEX: 2^%d = %s", + IPVX_MAX_PREFIX_LEN - 1, mpz_to_str10(IPVX_HALF_INDEX)); + log_debug("constraint", "IPvX_MAX_INDEX: 2^%d = %s", + IPVX_MAX_PREFIX_LEN, mpz_to_str10(IPVX_MAX_INDEX)); + log_debug("constraint", "IPvX RADIX_LENGTH: %d, RADIX_SIZE: 2^%d = %s", + RADIX_LENGTH, IPVX_MAX_PREFIX_LEN - RADIX_LENGTH, + mpz_to_str10(RADIX_SIZE)); + } +} + +// Initialize the tree. +// All addresses will initially have the given value. +constraint_t *constraint_init(const mpz_t value, size_t ipvx_max_len) { + log_debug("constraint", "initializing for IPvX, max-len=%d", ipvx_max_len); + + cal_ipvx_mask(ipvx_max_len); + constraint_t *con = xmalloc(sizeof(constraint_t)); + con->root = _create_leaf(value); + con->radix = xcalloc(sizeof(mpz_t), 1u << RADIX_LENGTH); + con->painted = 0; + + return con; +} + +// Free the subtree rooted at node. +static void _destroy_subtree(node_t *node) { + if (node == NULL) return; + + mpz_clear(node->value); + mpz_clear(node->count); + _destroy_subtree(node->l); + _destroy_subtree(node->r); + free(node); +} + +// Deinitialize and free the tree. +void constraint_free(constraint_t *con) { + assert(con); + + _destroy_subtree(con->root); + for (uint32_t i = 0; i < (1u << RADIX_LENGTH); i++) + mpz_clear(con->radix[i]); + mpz_clear(con->paint_value); + free(con); + mpz_clear(IPVX_HALF_INDEX); + mpz_clear(IPVX_MAX_INDEX); + mpz_clear(RADIX_SIZE); + + log_debug("constraint", "cleaning up"); +} + +// Convert from an internal node to a leaf. +static void _convert_to_leaf(node_t *node) { + assert(node); + assert(!IS_LEAF(node)); + + _destroy_subtree(node->l); + _destroy_subtree(node->r); + node->l = NULL; + node->r = NULL; +} + +// Recursive function to set value for a given network prefix within +// the tree. (Note: prefix must be in host byte order.) +static void _set_recurse(node_t *node, const mpz_t prefix, int len, + const mpz_t value) { + assert(node); + assert(0 <= len && len <= IPVX_MAX_PREFIX_LEN); + + // _print_ip(prefix, len); + if (len == 0) { + // We're at the end of the prefix; make this a leaf and set the + // value. + if (!IS_LEAF(node)) { + _convert_to_leaf(node); + } + mpz_set(node->value, value); + return; + } + + if (IS_LEAF(node)) { + // We're not at the end of the prefix, but we hit a leaf. + if (mpz_eq(node->value, value)) { + // A larger prefix has the same value, so we're done. + return; + } + // The larger prefix has a different value, so we need to + // convert it into an internal node and continue processing on + // one of the leaves. + node->l = _create_leaf(node->value); + node->r = _create_leaf(node->value); + } + + // We're not at the end of the prefix, and we're at an internal + // node. Recurse on the left or right subtree. + mpz_t flag, prefix_1; + mpz_init(flag); + mpz_and(flag, prefix, IPVX_HALF_INDEX); + mpz_init(prefix_1); + mpz_mul_ui(prefix_1, prefix, 2); + mpz_mod(prefix_1, prefix_1, IPVX_MAX_INDEX); + + if (mpz_not_zero(flag)) { + _set_recurse(node->r, prefix_1, len - 1, value); + } else { + _set_recurse(node->l, prefix_1, len - 1, value); + } + + mpz_clear(flag); + mpz_clear(prefix_1); + + // At this point, we're an internal node, and the value is set + // by one of our children or its descendent. If both children are + // leaves with the same value, we can discard them and become a left. + if (IS_LEAF(node->r) && IS_LEAF(node->l) && + mpz_eq(node->r->value, node->l->value)) { + mpz_set(node->value, node->l->value); + _convert_to_leaf(node); + } +} + +// Set the value for a given network prefix, overwriting any existing +// values on that prefix or subsets of it. +// (Note: prefix must be in host byte order.) +void constraint_set(constraint_t *con, const mpz_t prefix, int len, + const mpz_t value) { + assert(con); + + _set_recurse(con->root, prefix, len, value); + con->painted = 0; +} + +// Return the value pertaining to an address, according to the tree +// starting at given root. (Note: address must be in host byte order.) +static void _lookup_ipvx(mpz_t value, node_t *root, const mpz_t address) { + assert(root); + + node_t *node = root; + mpz_t flag, mask; + mpz_init(flag); + mpz_init_set(mask, IPVX_HALF_INDEX); + + for (;;) { + if (IS_LEAF(node)) { + mpz_set(value, node->value); + mpz_clear(flag); + mpz_clear(mask); + return; + } + mpz_and(flag, address, mask); + + if (mpz_not_zero(flag)) { + node = node->r; + } else { + node = node->l; + } + mpz_fdiv_q_ui(mask, mask, 2); + } +} + +// Return the value pertaining to an address. +// (Note: address must be in host byte order.) +void constraint_lookup_ipvx_for_value(mpz_t value, constraint_t *con, + const mpz_t ipvx) { + assert(con); + + _lookup_ipvx(value, con->root, ipvx); +} + +// Implement count_ips by recursing on halves of the tree. Size represents +// the number of addresses in a prefix at the current level of the tree. +// If paint is specified, each node will have its count set to the number of +// leaves under it set to value. +// If exclude_radix is specified, the number of addresses will exclude prefixes +// that are a /RADIX_LENGTH or larger +static void _count_ipvx_recurse(mpz_t count, node_t *node, const mpz_t value, + const mpz_t size, int paint, + int exclude_radix) { + assert(node); + + // log_debug("constraint", "_count_ips_recurse(size): %s", + // mpz_to_str10(size)); + if (IS_LEAF(node)) { + if (mpz_eq(node->value, value)) { + mpz_set(count, size); + // Exclude prefixes already included in the radix + if (exclude_radix && + mpz_ge(size, RADIX_SIZE)) { // save for length > RADIX_LENGTH + mpz_set_ui(count, 0); + } + } else { + mpz_set_ui(count, 0); + } + } else { // + + mpz_t count_l, count_r, size_2; + mpz_init(count_l); + mpz_init(count_r); + mpz_init(size_2); + mpz_fdiv_q_ui(size_2, size, 2); + _count_ipvx_recurse(count_l, node->l, value, size_2, paint, + exclude_radix); + _count_ipvx_recurse(count_r, node->r, value, size_2, paint, + exclude_radix); + mpz_add(count, count_l, count_r); + // log_debug("constraint", "_count_ipvx_recurse(count_l): %s", + // mpz_to_str10(count_l)); log_debug("constraint", + // "_count_ipvx_recurse(count_r): %s", mpz_to_str10(count_r)); + // log_debug("constraint", "_count_ipvx_recurse(count): %s", + // mpz_to_str10(count)); + + mpz_clear(count_l); + mpz_clear(count_r); + mpz_clear(size_2); + } + + if (paint) { + mpz_set(node->count, count); + } +} + +// Return the number of addresses that have a given value. +void constraint_count_ipvx_of_value(mpz_t count, constraint_t *con, + const mpz_t value) { + assert(con); + + if (con->painted && mpz_eq(con->paint_value, value)) { + // log_debug("constraint", + // "constraint_count_ipvx_of_value(painted)"); + mpz_t t; + mpz_init(t); + mpz_mul_ui(t, RADIX_SIZE, con->radix_len); + mpz_add(count, con->root->count, t); + mpz_clear(t); + + return; + } else { + // log_debug("constraint", + // "constraint_count_ipvx_of_value(recurse)"); + _count_ipvx_recurse(count, con->root, value, IPVX_MAX_INDEX, 0, 0); + } +} + +// Return a node that determines the values for the addresses with +// the given prefix. This is either the internal node that +// corresponds to the end of the prefix or a leaf node that +// encompasses the prefix. (Note: prefix must be in host byte order.) +static node_t *_lookup_node(node_t *root, const mpz_t prefix, int len) { + assert(root); + assert(0 <= len && len <= IPVX_MAX_PREFIX_LEN); + + node_t *node = root; + mpz_t flag, mask; + mpz_init(flag); + mpz_init_set(mask, IPVX_HALF_INDEX); + + int i; + for (i = 0; i < len; i++) { + if (IS_LEAF(node)) { + goto cleanup; + } + mpz_and(flag, prefix, mask); + if (mpz_not_zero(flag)) { + node = node->r; + } else { + node = node->l; + } + mpz_fdiv_q_ui(mask, mask, 2); + } + +cleanup: + mpz_clear(flag); + mpz_clear(mask); + + return node; +} + +// For each node, precompute the count of leaves beneath it set to value. +// Note that the tree can be painted for only one value at a time. +void constraint_paint_value(constraint_t *con, const mpz_t value) { + assert(con); + log_debug("constraint", "painting value %s", mpz_to_str10(value)); + + // Paint everything(count) except what we will put in radix + mpz_t count; + mpz_init(count); + _count_ipvx_recurse(count, con->root, value, IPVX_MAX_INDEX, 1, 1); + mpz_clear(count); + + // Fill in the radix array with a list of addresses + size_t i; + con->radix_len = 0; + mpz_t prefix; + mpz_init(prefix); + for (i = 0; i < (1u << RADIX_LENGTH); i++) { // /RADIX_LENGTH = /20 + mpz_mul_ui(prefix, RADIX_SIZE, i); + node_t *node = _lookup_node(con->root, prefix, RADIX_LENGTH); + if (IS_LEAF(node) && mpz_eq(node->value, value)) { + // Add this prefix to the radix + mpz_init_set(con->radix[con->radix_len++], prefix); + } + } + mpz_clear(prefix); + + mpz_t radix_c; + mpz_init(radix_c); + mpz_mul_ui(radix_c, RADIX_SIZE, con->radix_len); + log_debug( + "constraint", "%s IPvXs in radix array, %s IPvXs in tree, radix-len=%d", + mpz_to_str10(radix_c), mpz_to_str10(con->root->count), con->radix_len); + mpz_clear(radix_c); + + con->painted = 1; + mpz_set(con->paint_value, value); +} + +// Return the nth painted IP address. +static void _lookup_index(mpz_t ip, node_t *root, const mpz_t index) { + assert(root); + + node_t *node = root; + mpz_set_ui(ip, 0); + mpz_t mask, n; + mpz_init_set(mask, IPVX_HALF_INDEX); + mpz_init_set(n, index); + + for (;;) { + if (IS_LEAF(node)) { + mpz_ior(ip, ip, n); // prefix + mpz_clear(mask); + mpz_clear(n); + return; + } + if (mpz_lt(n, node->l->count)) { + node = node->l; + } else { + mpz_sub(n, n, node->l->count); + node = node->r; + mpz_ior(ip, ip, mask); + } + mpz_fdiv_q_ui(mask, mask, 2); + } +} + +// For a given value, return the IP address with zero-based index n. +// (i.e., if there are three addresses with value 0xFF, looking up index 1 +// will return the second one). +// Note that the tree must have been previously painted with this value. +void constraint_lookup_index_for_ipvx(mpz_t ipvx, constraint_t *con, + const mpz_t index, const mpz_t value) { + assert(con); + + if (!con->painted || mpz_ne(con->paint_value, value)) { + constraint_paint_value(con, value); + } + + mpz_t radix_idx; + mpz_init(radix_idx); + mpz_fdiv_q(radix_idx, index, RADIX_SIZE); + if (mpz_lt_ui(radix_idx, con->radix_len)) { + // Radix lookup + mpz_t radix_offset; + mpz_init(radix_offset); + mpz_mod(radix_offset, index, RADIX_SIZE); + mpz_ior(ipvx, con->radix[mpz_get_ui(radix_idx)], + radix_offset); // TODO radix_idx is no more than int32 + mpz_clear(radix_offset); + return; + } + mpz_clear(radix_idx); + + // Otherwise, do the "slow" lookup in tree. + // Note that tree counts do NOT include things in the radix, + // so we subtract these off here. + mpz_t t, n; + mpz_init(t); + mpz_mul_ui(t, RADIX_SIZE, con->radix_len); + mpz_init(n); + mpz_sub(n, index, t); + mpz_clear(t); + assert(mpz_lt(n, con->root->count)); + _lookup_index(ipvx, con->root, n); + mpz_clear(n); +} + +// ui using +constraint_t *constraint_init_ui(mpz_t_ui32 value, size_t ipvx_max_len) { + mpz_t value_m; + mpz_init_set_ui(value_m, value); + constraint_t *con = constraint_init(value_m, ipvx_max_len); + mpz_clear(value_m); + + return con; +} + +void constraint_set_ui(constraint_t *con, const mpz_t prefix, int len, + mpz_t_ui32 value) { + mpz_t value_m; + mpz_init_set_ui(value_m, value); + constraint_set(con, prefix, len, value_m); + mpz_clear(value_m); +} + +void constraint_paint_value_ui(constraint_t *con, mpz_t_ui32 value) { + mpz_t value_m; + mpz_init_set_ui(value_m, value); + constraint_paint_value(con, value_m); + mpz_clear(value_m); +} + +void constraint_lookup_index_for_ipvx_ui(mpz_t ipvx, constraint_t *con, + const mpz_t index, mpz_t_ui32 value) { + mpz_t value_m; + mpz_init_set_ui(value_m, value); + constraint_lookup_index_for_ipvx(ipvx, con, index, value_m); + mpz_clear(value_m); +} + +void constraint_count_ipvx_of_value_ui(mpz_t count, constraint_t *con, + mpz_t_ui32 value) { + mpz_t value_m; + mpz_init_set_ui(value_m, value); + constraint_count_ipvx_of_value(count, con, value_m); + mpz_clear(value_m); +} + +mpz_t_ui32 constraint_lookup_ipvx_for_value_ui(constraint_t *con, + const mpz_t ipvx) { + mpz_t value_m; + mpz_init(value_m); + constraint_lookup_ipvx_for_value(value_m, con, ipvx); + mpz_t_ui32 value = (mpz_t_ui32) mpz_get_ui(value_m); + mpz_clear(value_m); + + return value; +} + +// uint32_t compatible +void constraint_set_32(constraint_t *con, mpz_t_ui32 prefix, int len, + mpz_t_ui32 value) { + mpz_t prefix_m, value_m; + mpz_init_set_ui(prefix_m, prefix); + mpz_init_set_ui(value_m, value); + constraint_set(con, prefix_m, len, value_m); + mpz_clear(prefix_m); + mpz_clear(value_m); +} + +mpz_t_ui32 constraint_lookup_ipvx_for_value_32(constraint_t *con, + mpz_t_ui32 ipvx) { + mpz_t value_m, address_m; + mpz_init(value_m); + mpz_init_set_ui(address_m, ipvx); + constraint_lookup_ipvx_for_value(value_m, con, address_m); + mpz_t_ui32 value = (mpz_t_ui32) mpz_get_ui(value_m); + mpz_clear(value_m); + mpz_clear(address_m); + + return value; +} + +mpz_t_ui64 constraint_count_ipvx_of_value_32(constraint_t *con, + mpz_t_ui32 value) { + mpz_t count_m, value_m; + mpz_init(count_m); + mpz_init_set_ui(value_m, value); + constraint_count_ipvx_of_value(count_m, con, value_m); + mpz_t_ui64 count = (mpz_t_ui64) mpz_get_ui(count_m); + mpz_clear(value_m); + mpz_clear(count_m); + + return count; +} + +mpz_t_ui32 constraint_lookup_index_for_ipvx_32(constraint_t *con, + mpz_t_ui64 index, + mpz_t_ui32 value) { + mpz_t index_m, value_m, ipvx_m; + mpz_init_set_ui(index_m, index); + mpz_init_set_ui(value_m, value); + mpz_init(ipvx_m); + constraint_lookup_index_for_ipvx(ipvx_m, con, index_m, value_m); + mpz_t_ui32 ipvx = (mpz_t_ui32) mpz_get_ui(ipvx_m); + mpz_clear(index_m); + mpz_clear(value_m); + mpz_clear(ipvx_m); + + return ipvx; +} diff --git a/lib/constraint.h b/lib/constraint.h new file mode 100644 index 0000000..e476f88 --- /dev/null +++ b/lib/constraint.h @@ -0,0 +1,72 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_CONSTRAINT_H +#define XMAP_CONSTRAINT_H + +#include +#include + +#include "../lib/util.h" + +typedef struct _constraint constraint_t; +typedef uint64_t mpz_t_ui64; +typedef uint32_t mpz_t_ui32; + +// All addresses will initially have the given value. +constraint_t *constraint_init(const mpz_t value, size_t ipvx_max_len); + +void constraint_free(constraint_t *con); + +void constraint_set(constraint_t *con, const mpz_t prefix, int len, + const mpz_t value); + +void constraint_lookup_ipvx_for_value(mpz_t value, constraint_t *con, + const mpz_t ipvx); + +void constraint_count_ipvx_of_value(mpz_t count, constraint_t *con, + const mpz_t value); + +void constraint_paint_value(constraint_t *con, const mpz_t value); + +void constraint_lookup_index_for_ipvx(mpz_t ipvx, constraint_t *con, + const mpz_t index, const mpz_t value); + +// ui using +constraint_t *constraint_init_ui(mpz_t_ui32 value, size_t ipvx_max_len); + +void constraint_set_ui(constraint_t *con, const mpz_t prefix, int len, + mpz_t_ui32 value); + +void constraint_paint_value_ui(constraint_t *con, mpz_t_ui32 value); + +void constraint_lookup_index_for_ipvx_ui(mpz_t ipvx, constraint_t *con, + const mpz_t index, mpz_t_ui32 value); + +void constraint_count_ipvx_of_value_ui(mpz_t count, constraint_t *con, + mpz_t_ui32 value); + +mpz_t_ui32 constraint_lookup_ipvx_for_value_ui(constraint_t *con, + const mpz_t ipvx); + +// uint32_t compatible +void constraint_set_32(constraint_t *con, mpz_t_ui32 prefix, int len, + mpz_t_ui32 value); + +mpz_t_ui32 constraint_lookup_ipvx_for_value_32(constraint_t *con, + mpz_t_ui32 ipvx); + +mpz_t_ui64 constraint_count_ipvx_of_value_32(constraint_t *con, + mpz_t_ui32 value); + +mpz_t_ui32 constraint_lookup_index_for_ipvx_32(constraint_t *con, + mpz_t_ui64 index, + mpz_t_ui32 value); + +#endif // XMAP_CONSTRAINT_H diff --git a/lib/csv.c b/lib/csv.c new file mode 100644 index 0000000..9ce6b82 --- /dev/null +++ b/lib/csv.c @@ -0,0 +1,37 @@ +#include "csv.h" + +int csv_find_index(char *header, const char **names, size_t names_len) { + char *split = header; + for (int idx = 0; split != NULL; ++idx) { + char *front = (idx == 0) ? split : split + 1; + for (size_t i = 0; i < names_len; ++i) { + if (strncmp(front, names[i], strlen(names[i])) == 0) { + return idx; + } + } + split = strchr(front, ','); + } + + return -1; +} + +char *csv_get_index(char *row, size_t idx) { + char *split = row; + for (size_t i = 0; i < idx; ++i) { + split = strchr(split + 1, ','); + if (split == NULL) { + return NULL; + } + } + + char *entry; + char *start = (idx == 0) ? split : split + 1; + char *end = strchr(start, ','); + if (end != NULL) { + entry = strndup(start, end - start); + } else { + entry = strdup(start); + } + + return entry; +} diff --git a/lib/csv.h b/lib/csv.h new file mode 100644 index 0000000..0b149c2 --- /dev/null +++ b/lib/csv.h @@ -0,0 +1,10 @@ +#ifndef XMAP_CSV_H +#define XMAP_CSV_H + +#include + +int csv_find_index(char *header, const char **names, size_t names_len); + +char *csv_get_index(char *row, size_t idx); + +#endif // XMAP_CSV_H diff --git a/lib/gmp-ext.c b/lib/gmp-ext.c new file mode 100644 index 0000000..cb572a1 --- /dev/null +++ b/lib/gmp-ext.c @@ -0,0 +1,94 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "gmp-ext.h" + +#include +#include + +int mpz_eq(const mpz_t op1, const mpz_t op2) { return mpz_cmp(op1, op2) == 0; } + +int mpz_ne(const mpz_t op1, const mpz_t op2) { return mpz_cmp(op1, op2) != 0; } + +int mpz_ge(const mpz_t op1, const mpz_t op2) { return mpz_cmp(op1, op2) >= 0; } + +int mpz_le(const mpz_t op1, const mpz_t op2) { return mpz_cmp(op1, op2) <= 0; } + +int mpz_gt(const mpz_t op1, const mpz_t op2) { return mpz_cmp(op1, op2) > 0; } + +int mpz_lt(const mpz_t op1, const mpz_t op2) { return mpz_cmp(op1, op2) < 0; } + +int mpz_eq_ui(const mpz_t op1, unsigned long int op2) { + return mpz_cmp_ui(op1, op2) == 0; +} + +int mpz_ne_ui(const mpz_t op1, unsigned long int op2) { + return mpz_cmp_ui(op1, op2) != 0; +} + +int mpz_ge_ui(const mpz_t op1, unsigned long int op2) { + return mpz_cmp_ui(op1, op2) >= 0; +} + +int mpz_le_ui(const mpz_t op1, unsigned long int op2) { + return mpz_cmp_ui(op1, op2) <= 0; +} + +int mpz_gt_ui(const mpz_t op1, unsigned long int op2) { + return mpz_cmp_ui(op1, op2) > 0; +} + +int mpz_lt_ui(const mpz_t op1, unsigned long int op2) { + return mpz_cmp_ui(op1, op2) < 0; +} + +int mpz_not_zero(const mpz_t op) { return mpz_ne_ui(op, 0); } + +int mpz_zero(const mpz_t op) { return mpz_eq_ui(op, 0); } + +void mpz_to_uint8s(const mpz_t op, uint8_t *str, size_t bytes_len) { + size_t count; + uint8_t str_t[bytes_len]; + mpz_export(str_t, &count, 1, 1, 0, 0, op); + + if (count > bytes_len) count = bytes_len; + for (size_t i = 1; i <= count; i++) + str[bytes_len - i] = str_t[count - i]; +} + +void mpz_from_uint8s(mpz_t op, const uint8_t *str, size_t bytes_len) { + mpz_import(op, bytes_len, 1, 1, 0, 0, str); +} + +void mpz_to_uint8s_bits(const mpz_t op, uint8_t *str, size_t bits_len) { + mpz_t op_; + mpz_init_set(op_, op); + size_t shift_len = bits_len % 8; + if (shift_len) { + shift_len = 8 - shift_len; + mpz_mul_2exp(op_, op, shift_len); + } + + int bytes_len = (int) (ceil((bits_len) / (double) 8)); + mpz_to_uint8s(op_, str, bytes_len); + mpz_clear(op_); +} + +void mpz_from_uint8s_bits(mpz_t op, const uint8_t *str, size_t bits_len) { + int bytes_len = (int) (ceil((bits_len) / (double) 8)); + mpz_from_uint8s(op, str, bytes_len); + size_t shift_len = bits_len % 8; + + if (shift_len) { + shift_len = 8 - shift_len; + mpz_fdiv_q_2exp(op, op, shift_len); + } +} + +char *mpz_to_str10(const mpz_t op) { return mpz_get_str(NULL, 10, op); } diff --git a/lib/gmp-ext.h b/lib/gmp-ext.h new file mode 100644 index 0000000..4b580d7 --- /dev/null +++ b/lib/gmp-ext.h @@ -0,0 +1,54 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_GMP_EXT_H +#define XMAP_GMP_EXT_H + +#include +#include + +int mpz_eq(const mpz_t op1, const mpz_t op2); + +int mpz_ne(const mpz_t op1, const mpz_t op2); + +int mpz_ge(const mpz_t op1, const mpz_t op2); + +int mpz_le(const mpz_t op1, const mpz_t op2); + +int mpz_gt(const mpz_t op1, const mpz_t op2); + +int mpz_lt(const mpz_t op1, const mpz_t op2); + +int mpz_eq_ui(const mpz_t op1, unsigned long int op2); + +int mpz_ne_ui(const mpz_t op1, unsigned long int op2); + +int mpz_ge_ui(const mpz_t op1, unsigned long int op2); + +int mpz_le_ui(const mpz_t op1, unsigned long int op2); + +int mpz_gt_ui(const mpz_t op1, unsigned long int op2); + +int mpz_lt_ui(const mpz_t op1, unsigned long int op2); + +int mpz_not_zero(const mpz_t op); + +int mpz_zero(const mpz_t op); + +void mpz_to_uint8s(const mpz_t op, uint8_t *str, size_t bytes_len); + +void mpz_from_uint8s(mpz_t op, const uint8_t *str, size_t bytes_len); + +void mpz_to_uint8s_bits(const mpz_t op, uint8_t *str, size_t bits_len); + +void mpz_from_uint8s_bits(mpz_t op, const uint8_t *str, size_t bits_len); + +char *mpz_to_str10(const mpz_t op); + +#endif // XMAP_GMP_EXT_H diff --git a/lib/includes.h b/lib/includes.h new file mode 100644 index 0000000..7ce1e89 --- /dev/null +++ b/lib/includes.h @@ -0,0 +1,49 @@ +#ifndef XMAP_INCLUDES_H +#define XMAP_INCLUDES_H + +#ifdef __APPLE__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" + +#include + +#pragma GCC diagnostic warning "-Wflexible-array-extensions" +#endif + +#ifndef __FAVOR_BSD +#define __FAVOR_BSD 2 +#endif +#ifndef __USE_BSD +#define __USE_BSD +#endif + +#include +#include +#include +#include + +#ifndef __APPLE__ +#include +#endif + +#include +#include +#include + +#if defined(__NetBSD__) +#define ICMP_UNREACH_PRECEDENCE_CUTOFF ICMP_UNREACH_PREC_CUTOFF +#include +#else + +#include + +#endif + +#include +#include // NOTE: net/if.h MUST be included BEFORE ifaddrs.h +#include +#include + +#define MAC_ADDR_LEN ETHER_ADDR_LEN +#define UNUSED __attribute__((unused)) + +#endif // XMAP_INCLUDES_H diff --git a/lib/lockfd.c b/lib/lockfd.c new file mode 100644 index 0000000..eb59536 --- /dev/null +++ b/lib/lockfd.c @@ -0,0 +1,42 @@ +#include "lockfd.h" + +#include +#include + +#include "xalloc.h" + +static pthread_mutex_t **mutexes = NULL; + +static pthread_mutex_t *get_mutex(int fd) { + assert(fd < 3 && "todo: implement generically"); + + if (!mutexes) { + mutexes = xmalloc(3 * sizeof(char *)); + assert(mutexes); + } + + if (!mutexes[fd]) { + mutexes[fd] = xmalloc(sizeof(pthread_mutex_t)); + assert(mutexes[fd]); + pthread_mutex_init(mutexes[fd], NULL); // create mutex + assert(mutexes[fd]); + } + + return mutexes[fd]; +} + +int lock_fd(int fd) { return pthread_mutex_lock(get_mutex(fd)); } + +int unlock_fd(int fd) { return pthread_mutex_unlock(get_mutex(fd)); } + +int lock_file(FILE *f) { + assert(f); + + return lock_fd(fileno(f)); +} + +int unlock_file(FILE *f) { + assert(f); + + return unlock_fd(fileno(f)); +} diff --git a/lib/lockfd.h b/lib/lockfd.h new file mode 100644 index 0000000..ec76278 --- /dev/null +++ b/lib/lockfd.h @@ -0,0 +1,14 @@ +#ifndef XMAP_LOCKFD_H +#define XMAP_LOCKFD_H + +#include + +int lock_fd(int fd); + +int unlock_fd(int fd); + +int lock_file(FILE *f); + +int unlock_file(FILE *f); + +#endif // XMAP_LOCKFD_H diff --git a/lib/logger.c b/lib/logger.c new file mode 100644 index 0000000..93c1e26 --- /dev/null +++ b/lib/logger.c @@ -0,0 +1,271 @@ +/* + * Logger Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "logger.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "lockfd.h" +#include "xalloc.h" + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static enum LogLevel log_output_level = XLOG_INFO; + +static FILE *log_output_stream = NULL; +static int color = 0; + +static int log_to_syslog = 0; + +static const char *log_level_name[] = {"FATAL", "ERROR", "WARN", + "INFO", "DEBUG", "TRACE"}; + +#define RED "\x1b[31m" +#define GREEN "\x1b[32m" +#define YELLOW "\x1b[33m" +#define BLUE "\x1b[34m" +#define MAGENTA "\x1b[35m" +#define CYAN "\x1b[36m" +#define RESET "\033[0m" + +#define COLOR(x) \ + do { \ + if (color) fprintf(log_output_stream, "%s", x); \ + } while (0) + +static const char *color_for_level(enum LogLevel level) { + switch (level) { + case XLOG_FATAL: + return RED; + case XLOG_ERROR: + return MAGENTA; + case XLOG_WARN: + return YELLOW; + case XLOG_INFO: + return GREEN; + case XLOG_DEBUG: + return BLUE; + case XLOG_TRACE: + return RESET; + default: + return RESET; + } +} + +// basic log output function +static int LogLogVA(enum LogLevel level, const char *loggerName, + const char *logMessage, va_list args) { + if (level <= log_output_level) { + if (!log_output_stream) { + log_output_stream = stderr; + } + // if logging to a shared output channel, then use a global + // lock across XMap. Otherwise, if we're logging to a file, + // only lockin with the module, in order to avoid having + // corrupt log entries. + if (log_output_stream == stdout || log_output_stream == stderr) { + lock_file(log_output_stream); + } else { + pthread_mutex_lock(&mutex); + } + if (color) { + COLOR(color_for_level(level)); + } + + const char * levelName = log_level_name[level]; + struct timeval now; + char timestamp[256]; + gettimeofday(&now, NULL); + time_t sec = now.tv_sec; + struct tm *ptm = localtime(&sec); + strftime(timestamp, 20, "%b %d %H:%M:%S", ptm); + fprintf(log_output_stream, "%s.%03ld [%s] ", timestamp, + (long) now.tv_usec / 1000, levelName); + if (loggerName) { + fprintf(log_output_stream, "%s: ", loggerName); + } + if (logMessage) { + vfprintf(log_output_stream, logMessage, args); + } + if (loggerName || logMessage) { + fputs("\n", log_output_stream); + } + if (color) { + COLOR(RESET); + } + fflush(log_output_stream); + if (log_output_stream == stdout || log_output_stream == stderr) { + unlock_file(log_output_stream); + } else { + pthread_mutex_unlock(&mutex); + } + } + + return EXIT_SUCCESS; +} + +int log_fatal(const char *name, const char *message, ...) { + va_list va; + va_start(va, message); + LogLogVA(XLOG_FATAL, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_CRIT), message, va); + va_end(va); + } + + exit(EXIT_FAILURE); +} + +int log_error(const char *name, const char *message, ...) { + va_list va; + va_start(va, message); + int ret = LogLogVA(XLOG_ERROR, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_ERR), message, va); + va_end(va); + } + + return ret; +} + +int log_warn(const char *name, const char *message, ...) { + va_list va; + va_start(va, message); + int ret = LogLogVA(XLOG_WARN, name, message, va); + va_end(va); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_WARNING), message, va); + va_end(va); + } + + return ret; +} + +int log_info(const char *name, const char *message, ...) { + va_list va; + va_start(va, message); + int ret = LogLogVA(XLOG_INFO, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_INFO), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} + +int log_debug(const char *name, const char *message, ...) { + va_list va; + va_start(va, message); + int ret = LogLogVA(XLOG_DEBUG, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_DEBUG), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} + +#ifdef DEBUG +extern int log_trace(const char *name, const char *message, ...) { + va_list va; + va_start(va, message); + int ret = LogLogVA(XLOG_TRACE, name, message, va); + va_end(va); + + char *prefixed = xmalloc(strlen(name) + strlen(message) + 3); + strcpy(prefixed, name); + strcat(prefixed, ": "); + strcat(prefixed, message); + + if (log_to_syslog) { + va_start(va, message); + vsyslog(LOG_MAKEPRI(LOG_USER, LOG_DEBUG), prefixed, va); + va_end(va); + } + + free(prefixed); + + return ret; +} +#endif + +int log_init(FILE *stream, enum LogLevel level, int syslog_enabled, + const char *appname) { + log_output_stream = stream; + log_output_level = level; + + if (syslog_enabled) { + log_to_syslog = 1; + openlog(appname, 0, LOG_USER); // no options + } + + if (isatty(fileno(log_output_stream))) { + color = 1; + } + + return 0; +} + +void check_and_log_file_error(FILE *file, const char *name) { + if (ferror(file)) { + log_fatal(name, "unable to write to file"); + } +} + +double now(void) { + struct timeval now; + gettimeofday(&now, NULL); + + return (double) now.tv_sec + (double) now.tv_usec / 1000000.; +} + +size_t dstrftime(char *buf, size_t maxsize, const char *format, double tm) { + struct timeval tv; + double tm_floor; + tm_floor = floor(tm); + tv.tv_sec = (long) tm_floor; + tv.tv_usec = (long) (tm - floor(tm)) * 1000000; + + return strftime(buf, maxsize, format, localtime((const time_t *) &tv)); +} diff --git a/lib/logger.h b/lib/logger.h new file mode 100644 index 0000000..aef9a70 --- /dev/null +++ b/lib/logger.h @@ -0,0 +1,63 @@ +/* + * Logger Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_LOGGER_H +#define XMAP_LOGGER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// do not collide with constants defined in syslog.h +enum LogLevel { + XLOG_FATAL, + XLOG_ERROR, + XLOG_WARN, + XLOG_INFO, + XLOG_DEBUG, + XLOG_TRACE, + XNUM_LOGLEVELS +}; + +int log_fatal(const char *loggerName, const char *logMessage, ...) + __attribute__((noreturn)); + +int log_error(const char *loggerName, const char *logMessage, ...); + +int log_warn(const char *loggerName, const char *logMessage, ...); + +int log_info(const char *loggerName, const char *logMessage, ...); + +int log_debug(const char *loggerName, const char *logMessage, ...); + +#ifdef DEBUG +int log_trace(const char *loggerName, const char *logMessage, ...); +#else +#define log_trace(...) ; +#endif + +int log_init(FILE *stream, enum LogLevel level, int syslog_enabled, + const char *syslog_app); + +void check_and_log_file_error(FILE *file, const char *name); + +size_t dstrftime(char *, size_t, const char *, double); + +double now(); + +#ifdef __cplusplus +}; +#endif + +#endif // XMAP_LOGGER_H diff --git a/lib/random.c b/lib/random.c new file mode 100644 index 0000000..6cf7da6 --- /dev/null +++ b/lib/random.c @@ -0,0 +1,32 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "random.h" + +#include +#include +#include + +#include "logger.h" + +#define RANDSRC "/dev/urandom" + +int random_bytes(void *dst, size_t n) { + FILE *f = fopen(RANDSRC, "rb"); + if (!f) { + log_fatal("random", "unable to read /dev/urandom: %s", strerror(errno)); + } + + size_t r = fread(dst, n, 1, f); + fclose(f); + if (r < 1) { + return 0; + } + + return 1; +} diff --git a/lib/random.h b/lib/random.h new file mode 100644 index 0000000..6ed470f --- /dev/null +++ b/lib/random.h @@ -0,0 +1,9 @@ +#ifndef XMAP_RANDOM_H +#define XMAP_RANDOM_H + +#include +#include + +int random_bytes(void *dst, size_t n); + +#endif // XMAP_RANDOM_H diff --git a/lib/redis.c b/lib/redis.c new file mode 100644 index 0000000..d97fedb --- /dev/null +++ b/lib/redis.c @@ -0,0 +1,429 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "redis.h" + +#include +#include +#include +#include +#include + +#include + +#include "logger.h" +#include "xalloc.h" + +#define REDIS_TIMEOUT 2 + +#undef MIN +#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) + +int redis_parse_connstr(char *connstr, redisconf_t *redis_conf) { + memset(redis_conf->error, 0, PMAP_REDIS_ERRLEN); + if (!strncmp("tcp://", connstr, 6)) { + // Zero-out the error message + char *servername = xmalloc(strlen(connstr)); + char *list_name = xmalloc(strlen(connstr)); + uint32_t port; + if (sscanf(connstr, "tcp://%[^:]:%u/%s", servername, &port, list_name) != + 3) { + char *back = + stpncpy(&redis_conf->error[0], + "unable to " + "parse redis connection string. This should be of the form " + "tcp://server:port/list-name for TCP connections. All fields" + " are required.", + PMAP_REDIS_ERRLEN); + *back = '\0'; + return PMAP_REDIS_ERROR; + } + redis_conf->type = T_TCP; + redis_conf->server = servername; + redis_conf->port = port; + redis_conf->list_name = list_name; + redis_conf->path = NULL; + } else if (!strncmp("local://", connstr, 8)) { + // looking for something along the lines of + // local:///tmp/redis.sock/list-name + // or local:///tmp/redis.sock/ + char *path = xmalloc(strlen(connstr)); + char *list_name = xmalloc(strlen(connstr)); + connstr = connstr + (size_t) 8; + char *listname = strrchr(connstr, '/'); + if (listname == NULL) { + char *back = + stpncpy(&redis_conf->error[0], "bad local url (missing a slash)", + PMAP_REDIS_ERRLEN); + *back = '\0'; + return PMAP_REDIS_ERROR; + } + + // Check if we have a list or not + listname += 1; + if (*listname != '\0') { + redis_conf->list_name = list_name; + } else { + redis_conf->list_name = NULL; + } + + // Get the hostname + connstr[strrchr(connstr, '/') - connstr] = '\0'; + strcpy(path, connstr); + strcpy(list_name, listname); + redis_conf->type = T_LOCAL; + redis_conf->path = path; + redis_conf->server = NULL; + redis_conf->port = 0; + } else { + char *back = stpncpy(&redis_conf->error[0], + "redis connection string does not being with " + "tcp:// or local://", + PMAP_REDIS_ERRLEN); + *back = '\0'; + return PMAP_REDIS_ERROR; + } + return PMAP_REDIS_SUCCESS; +} + +redisContext *redis_connect(char *connstr) { + assert(connstr); + redisconf_t rconf; + redisconf_t *c = &rconf; + // handle old behavior where we only connected to a specific + // socket that we #defined. + if (!connstr) { + c->type = T_LOCAL; + c->path = strdup("/tmp/redis.sock"); + } else { + int retv = redis_parse_connstr(connstr, c); + log_error("redis", "Could not connect: %s", c->error); + if (retv != PMAP_REDIS_ERROR) { + return NULL; + } + } + return redis_connect_from_conf(c); +} + +redisContext *redis_connect_from_conf(redisconf_t *c) { + assert(c); + struct timeval timeout; + timeout.tv_sec = REDIS_TIMEOUT; + timeout.tv_usec = 0; + if (c->type == T_LOCAL) { + return (redisContext *) redisConnectUnixWithTimeout(c->path, timeout); + } else { + return (redisContext *) redisConnectWithTimeout(c->server, c->port, timeout); + } +} + +int redis_close(redisContext *rctx) { + assert(rctx); + redisFree(rctx); + return 0; +} + +static int chkerr(redisContext *rctx, redisReply *reply) { + assert(rctx); + if (reply == NULL || reply->type == REDIS_REPLY_ERROR) { + log_error("redis", "an error occurred when retrieving item from redis: %s", + rctx->errstr); + if (reply) { + freeReplyObject(reply); + } + return PMAP_REDIS_ERROR; + } + return 0; +} + +int redis_flush(redisContext *rctx) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "FLUSHDB"); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + freeReplyObject(reply); + return 0; +} + +int redis_existconf(redisContext *rctx, const char *name) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "EXISTS %s", name); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + int v = reply->integer; + freeReplyObject(reply); + return v; +} + +int redis_delconf(redisContext *rctx, const char *name) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "DEL %s", name); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + freeReplyObject(reply); + return 0; +} + +int redis_setconf(redisContext *rctx, const char *name, char *value) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "SET %s %s", name, value); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + freeReplyObject(reply); + return 0; +} + +int redis_getconf(redisContext *rctx, const char *name, char *buf, + size_t maxlen) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "GET %s", name); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + strncpy(buf, reply->str, maxlen - 1); + buf[maxlen - 1] = '\0'; + freeReplyObject(reply); + return 0; +} + +uint32_t redis_getconf_uint32_t(redisContext *rctx, const char *key) { + assert(rctx); + char buf[50]; + assert(redis_getconf(rctx, key, buf, 50) == 0); + return (uint32_t) atoi(buf); +} + +int redis_setconf_uint32_t(redisContext *rctx, const char *key, + uint32_t value) { + assert(rctx); + char buf[50]; + sprintf(buf, "%u", value); + return redis_setconf(rctx, key, buf); +} + +static long redis_get_sizeof(redisContext *rctx, const char *cmd, + const char *name) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "%s %s", cmd, name); + assert(reply); + assert(reply->type == REDIS_REPLY_INTEGER); + long rtr = reply->integer; + freeReplyObject(reply); + return rtr; +} + +long redis_get_sizeof_list(redisContext *rctx, const char *name) { + return redis_get_sizeof(rctx, "LLEN", name); +} + +long redis_get_sizeof_set(redisContext *rctx, const char *name) { + return redis_get_sizeof(rctx, "SCARD", name); +} + +int redis_pull(redisContext *rctx, char *redisqueuename, void *buf, int maxload, + size_t obj_size, int *numloaded, const char *cmd) { + assert(rctx); + long elems_in_redis = redis_get_sizeof_list(rctx, redisqueuename); + long num_to_add = MIN(elems_in_redis, maxload); + log_debug("redis", + "redis load called on %s. Transferring %li of %li elements " + "to in-memory queue.", + redisqueuename, num_to_add, elems_in_redis); + for (int i = 0; i < num_to_add; i++) { + redisAppendCommand(rctx, "%s %s", cmd, redisqueuename); + } + for (int i = 0; i < num_to_add; i++) { + redisReply *reply = NULL; + int rc = redisGetReply(rctx, (void **) &reply); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + if (rc != REDIS_OK || reply == NULL) { + log_error("redis", "unknown error, could not get reply"); + if (reply) { + freeReplyObject(reply); + } + return PMAP_REDIS_ERROR; + } + if (reply->type != REDIS_REPLY_STRING) { + log_error("redis", "unxpected reply type from redis"); + freeReplyObject(reply); + return PMAP_REDIS_ERROR; + } + if ((size_t) reply->len != obj_size) { + freeReplyObject(reply); + log_error("redis", "response object length mismatch"); + return PMAP_REDIS_ERROR; + } + memcpy((void *) ((intptr_t) buf + i * obj_size), reply->str, obj_size); + freeReplyObject(reply); + *numloaded = i + 1; + } + return PMAP_REDIS_SUCCESS; +} + +int redis_lpull(redisContext *rctx, char *redisqueuename, void *buf, + int maxload, size_t obj_size, int *numloaded) { + return redis_pull(rctx, redisqueuename, buf, maxload, obj_size, numloaded, + "LPOP"); +} + +int redis_spull(redisContext *rctx, char *redisqueuename, void *buf, + int maxload, size_t obj_size, int *numloaded) { + return redis_pull(rctx, redisqueuename, buf, maxload, obj_size, numloaded, + "SRAND"); +} + +static int redis_pull_one(redisContext *rctx, char *queuename, void **buf, + size_t *len, const char *cmd) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "%s %s", cmd, queuename); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + if (reply->type == REDIS_REPLY_NIL) { + freeReplyObject(reply); + return PMAP_REDIS_EMPTY; + } + assert(reply->type == REDIS_REPLY_STRING); + *len = reply->len; + void *temp = (char *) malloc(*len); + assert(temp); + *buf = temp; + memcpy(temp, reply->str, *len); + freeReplyObject(reply); + return PMAP_REDIS_SUCCESS; +} + +int redis_lpull_one(redisContext *rctx, char *queuename, void **buf, + size_t *len) { + return redis_pull_one(rctx, queuename, buf, len, "LPOP"); +} + +int redis_spull_one(redisContext *rctx, char *queuename, void **buf, + size_t *len) { + return redis_pull_one(rctx, queuename, buf, len, "SRAND"); +} + +static int redis_push(redisContext *rctx, char *redisqueuename, void *buf, + int num, size_t len, const char *cmd) { + assert(rctx); + for (int i = 0; i < num; i++) { + void *load = (void *) ((intptr_t) buf + i * len); + int rc = + redisAppendCommand(rctx, "%s %s %b", cmd, redisqueuename, load, len); + if (rc != REDIS_OK || rctx->err) { + log_error("redis", "%s", rctx->errstr); + return PMAP_REDIS_ERROR; + } + } + redisReply *reply = NULL; + for (int i = 0; i < num; i++) { + int rc = redisGetReply(rctx, (void **) &reply); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + if (rc != REDIS_OK || reply == NULL) { + if (reply) { + freeReplyObject(reply); + } + return PMAP_REDIS_ERROR; + } + if (reply->type == REDIS_REPLY_ERROR) { + log_error("redis", "%s", rctx->errstr); + freeReplyObject(reply); + return PMAP_REDIS_ERROR; + } + freeReplyObject(reply); + } + return PMAP_REDIS_SUCCESS; +} + +int redis_lpush(redisContext *rctx, char *redisqueuename, void *buf, int num, + size_t len) { + return redis_push(rctx, redisqueuename, buf, num, len, "LPUSH"); +} + +int redis_rpush(redisContext *rctx, char *redisqueuename, void *buf, int num, + size_t len) { + return redis_push(rctx, redisqueuename, buf, num, len, "RPUSH"); +} + +int redis_spush(redisContext *rctx, char *redisqueuename, void *buf, int num, + size_t len) { + return redis_push(rctx, redisqueuename, buf, num, len, "SADD"); +} + +static int redis_push_one(redisContext *rctx, char *queuename, void *buf, + size_t len, const char *cmd) { + assert(rctx); + redisReply *reply = redisCommand(rctx, "%s %s %b", cmd, queuename, buf, len); + if (chkerr(rctx, reply)) { + return PMAP_REDIS_ERROR; + } + freeReplyObject(reply); + return PMAP_REDIS_SUCCESS; +} + +int redis_lpush_one(redisContext *rctx, char *queuename, void *buf, + size_t len) { + return redis_push_one(rctx, queuename, buf, len, "LPUSH"); +} + +int redis_spush_one(redisContext *rctx, char *queuename, void *buf, + size_t len) { + return redis_push_one(rctx, queuename, buf, len, "SADD"); +} + +static int redis_push_strings(redisContext *rctx, char *redisqueuename, + char **buf, int num, const char *cmd) { + assert(rctx); + for (int i = 0; i < num; i++) { + int rc = redisAppendCommand(rctx, "%s %s %s", cmd, redisqueuename, buf[i]); + if (rc != REDIS_OK || rctx->err) { + log_error("redis", "%s", rctx->errstr); + return PMAP_REDIS_ERROR; + } + } + redisReply *reply = NULL; + for (int i = 0; i < num; i++) { + if (redisGetReply(rctx, (void **) &reply) != REDIS_OK || rctx->err) { + log_error("redis", "%s", rctx->errstr); + if (reply) { + freeReplyObject(reply); + } + return PMAP_REDIS_ERROR; + } + if (reply->type == REDIS_REPLY_ERROR) { + log_error("redis", "%s", rctx->errstr); + freeReplyObject(reply); + return PMAP_REDIS_ERROR; + } + freeReplyObject(reply); + } + return PMAP_REDIS_SUCCESS; +} + +int redis_lpush_strings(redisContext *rctx, char *redisqueuename, char **buf, + int num) { + return redis_push_strings(rctx, redisqueuename, buf, num, "LPUSH"); +} + +int redis_rpush_strings(redisContext *rctx, char *redisqueuename, char **buf, + int num) { + return redis_push_strings(rctx, redisqueuename, buf, num, "RPUSH"); +} + +int redis_spush_strings(redisContext *rctx, char *redisqueuename, char **buf, + int num) { + return redis_push_strings(rctx, redisqueuename, buf, num, "SADD"); +} diff --git a/lib/redis.h b/lib/redis.h new file mode 100644 index 0000000..d1c5a8a --- /dev/null +++ b/lib/redis.h @@ -0,0 +1,91 @@ +#ifndef XMAP_REDIS_H +#define XMAP_REDIS_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define T_TCP 0 +#define T_LOCAL 1 + +#define PMAP_REDIS_SUCCESS 0 +#define PMAP_REDIS_EMPTY 1 +#define PMAP_REDIS_ERROR -1 + +#define PMAP_REDIS_ERRLEN 1024 + +typedef struct redisconf { + int type; + char *path; + char *server; + uint16_t port; + char *list_name; + char error[PMAP_REDIS_ERRLEN]; +} redisconf_t; + +int redis_parse_connstr(char *connstr, redisconf_t *redis_conf); + +int redis_existconf(redisContext *, const char *); + +int redis_flush(redisContext *); + +int redis_delconf(redisContext *, const char *); + +int redis_setconf(redisContext *, const char *, char *); + +int redis_getconf(redisContext *, const char *, char *, size_t); + +long redis_get_sizeof_list(redisContext *, const char *); + +long redis_get_sizeof_set(redisContext *, const char *); + +int redis_lpush(redisContext *, char *, void *, int, size_t); + +int redis_rpush(redisContext *, char *, void *, int, size_t); + +int redis_lpull(redisContext *, char *, void *, int, size_t, int *); + +int redis_spull(redisContext *, char *, void *, int, size_t, int *); + +int redis_spush(redisContext *, char *, void *, int, size_t); + +int redis_lpull_one(redisContext *rctx, char *queuename, void **buf, + size_t *len); + +int redis_spull_one(redisContext *rctx, char *queuename, void **buf, + size_t *len); + +int redis_lpush_one(redisContext *rctx, char *queuename, void *buf, size_t len); + +int redis_rpush_one(redisContext *rctx, char *queuename, void *buf, size_t len); + +int redis_spush_one(redisContext *rctx, char *queuename, void *buf, size_t len); + +int redis_lpush_strings(redisContext *rctx, char *redisqueuename, char **buf, + int num); + +int redis_rpush_strings(redisContext *rctx, char *redisqueuename, char **buf, + int num); + +int redis_spush_strings(redisContext *rctx, char *redisqueuename, char **buf, + int num); + +redisContext *redis_connect(char *connstr); + +redisContext *redis_connect_from_conf(redisconf_t *rconf); + +int redis_close(redisContext *rctx); + +uint32_t redis_getconf_uint32_t(redisContext *, const char *); + +int redis_setconf_uint32_t(redisContext *, const char *, uint32_t); + +#ifdef __cplusplus +} +#endif + +#endif //XMAP_REDIS_H diff --git a/lib/rijndael-alg-fst.c b/lib/rijndael-alg-fst.c new file mode 100644 index 0000000..91b57ff --- /dev/null +++ b/lib/rijndael-alg-fst.c @@ -0,0 +1,1270 @@ +/** + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "rijndael-alg-fst.h" + +/* +Te0[x] = S [x].[02, 01, 01, 03]; +Te1[x] = S [x].[03, 02, 01, 01]; +Te2[x] = S [x].[01, 03, 02, 01]; +Te3[x] = S [x].[01, 01, 03, 02]; +Te4[x] = S [x].[01, 01, 01, 01]; + +Td0[x] = Si[x].[0e, 09, 0d, 0b]; +Td1[x] = Si[x].[0b, 0e, 09, 0d]; +Td2[x] = Si[x].[0d, 0b, 0e, 09]; +Td3[x] = Si[x].[09, 0d, 0b, 0e]; +Td4[x] = Si[x].[01, 01, 01, 01]; +*/ + +static const u32 Te0[256] = { + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, 0xfff2f20dU, + 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, 0x60303050U, 0x02010103U, + 0xce6767a9U, 0x562b2b7dU, 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, + 0xec76769aU, 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, 0x41adadecU, + 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, 0x239c9cbfU, 0x53a4a4f7U, + 0xe4727296U, 0x9bc0c05bU, 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, + 0x4c26266aU, 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, 0xe2717193U, + 0xabd8d873U, 0x62313153U, 0x2a15153fU, 0x0804040cU, 0x95c7c752U, + 0x46232365U, 0x9dc3c35eU, 0x30181828U, 0x379696a1U, 0x0a05050fU, + 0x2f9a9ab5U, 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, 0x1209091bU, + 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, 0x361b1b2dU, 0xdc6e6eb2U, + 0xb45a5aeeU, 0x5ba0a0fbU, 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, + 0x7db3b3ceU, 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, 0x40202060U, + 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, 0xd46a6abeU, 0x8dcbcb46U, + 0x67bebed9U, 0x7239394bU, 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, + 0x85cfcf4aU, 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, 0x8a4545cfU, + 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, 0xa05050f0U, 0x783c3c44U, + 0x259f9fbaU, 0x4ba8a8e3U, 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, + 0x058f8f8aU, 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, 0x20101030U, + 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, 0x81cdcd4cU, 0x180c0c14U, + 0x26131335U, 0xc3ecec2fU, 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, + 0x2e171739U, 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, 0xc06060a0U, + 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, 0x44222266U, 0x542a2a7eU, + 0x3b9090abU, 0x0b888883U, 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, + 0x2814143cU, 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, 0x924949dbU, + 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, 0x9fc2c25dU, 0xbdd3d36eU, + 0x43acacefU, 0xc46262a6U, 0x399191a8U, 0x319595a4U, 0xd3e4e437U, + 0xf279798bU, 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, 0xd86c6cb4U, + 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, 0xca6565afU, 0xf47a7a8eU, + 0x47aeaee9U, 0x10080818U, 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, + 0x5c2e2e72U, 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, 0x964b4bddU, + 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, 0xe0707090U, 0x7c3e3e42U, + 0x71b5b5c4U, 0xcc6666aaU, 0x904848d8U, 0x06030305U, 0xf7f6f601U, + 0x1c0e0e12U, 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, 0xd9e1e138U, + 0xebf8f813U, 0x2b9898b3U, 0x22111133U, 0xd26969bbU, 0xa9d9d970U, + 0x078e8e89U, 0x339494a7U, 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, + 0xc9e9e920U, 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, 0x65bfbfdaU, + 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, 0x824141c3U, 0x299999b0U, + 0x5a2d2d77U, 0x1e0f0f11U, 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, + 0x2c16163aU, +}; +static const u32 Te1[256] = { + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, 0x0dfff2f2U, + 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, 0x50603030U, 0x03020101U, + 0xa9ce6767U, 0x7d562b2bU, 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, + 0x9aec7676U, 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, 0xec41adadU, + 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, 0xbf239c9cU, 0xf753a4a4U, + 0x96e47272U, 0x5b9bc0c0U, 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, + 0x6a4c2626U, 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, 0x93e27171U, + 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, 0x0c080404U, 0x5295c7c7U, + 0x65462323U, 0x5e9dc3c3U, 0x28301818U, 0xa1379696U, 0x0f0a0505U, + 0xb52f9a9aU, 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, 0x1b120909U, + 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, 0x2d361b1bU, 0xb2dc6e6eU, + 0xeeb45a5aU, 0xfb5ba0a0U, 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, + 0xce7db3b3U, 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, 0x60402020U, + 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, 0xbed46a6aU, 0x468dcbcbU, + 0xd967bebeU, 0x4b723939U, 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, + 0x4a85cfcfU, 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, 0xcf8a4545U, + 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, 0xf0a05050U, 0x44783c3cU, + 0xba259f9fU, 0xe34ba8a8U, 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, + 0x8a058f8fU, 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, 0x30201010U, + 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, 0x4c81cdcdU, 0x14180c0cU, + 0x35261313U, 0x2fc3ececU, 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, + 0x392e1717U, 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, 0xa0c06060U, + 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, 0x66442222U, 0x7e542a2aU, + 0xab3b9090U, 0x830b8888U, 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, + 0x3c281414U, 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, 0xdb924949U, + 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, 0x5d9fc2c2U, 0x6ebdd3d3U, + 0xef43acacU, 0xa6c46262U, 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, + 0x8bf27979U, 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, 0xb4d86c6cU, + 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, 0xafca6565U, 0x8ef47a7aU, + 0xe947aeaeU, 0x18100808U, 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, + 0x725c2e2eU, 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, 0xdd964b4bU, + 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, 0x90e07070U, 0x427c3e3eU, + 0xc471b5b5U, 0xaacc6666U, 0xd8904848U, 0x05060303U, 0x01f7f6f6U, + 0x121c0e0eU, 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, 0x38d9e1e1U, + 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, 0xbbd26969U, 0x70a9d9d9U, + 0x89078e8eU, 0xa7339494U, 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, + 0x20c9e9e9U, 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, 0xda65bfbfU, + 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, 0xc3824141U, 0xb0299999U, + 0x775a2d2dU, 0x111e0f0fU, 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, + 0x3a2c1616U, +}; +static const u32 Te2[256] = { + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, 0xf20dfff2U, + 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, 0x30506030U, 0x01030201U, + 0x67a9ce67U, 0x2b7d562bU, 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, + 0x769aec76U, 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, 0xadec41adU, + 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, 0x9cbf239cU, 0xa4f753a4U, + 0x7296e472U, 0xc05b9bc0U, 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, + 0x266a4c26U, 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, 0x7193e271U, + 0xd873abd8U, 0x31536231U, 0x153f2a15U, 0x040c0804U, 0xc75295c7U, + 0x23654623U, 0xc35e9dc3U, 0x18283018U, 0x96a13796U, 0x050f0a05U, + 0x9ab52f9aU, 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, 0x091b1209U, + 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, 0x1b2d361bU, 0x6eb2dc6eU, + 0x5aeeb45aU, 0xa0fb5ba0U, 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, + 0xb3ce7db3U, 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, 0x20604020U, + 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, 0x6abed46aU, 0xcb468dcbU, + 0xbed967beU, 0x394b7239U, 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, + 0xcf4a85cfU, 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, 0x45cf8a45U, + 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, 0x50f0a050U, 0x3c44783cU, + 0x9fba259fU, 0xa8e34ba8U, 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, + 0x8f8a058fU, 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, 0x10302010U, + 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, 0xcd4c81cdU, 0x0c14180cU, + 0x13352613U, 0xec2fc3ecU, 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, + 0x17392e17U, 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, 0x60a0c060U, + 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, 0x22664422U, 0x2a7e542aU, + 0x90ab3b90U, 0x88830b88U, 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, + 0x143c2814U, 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, 0x49db9249U, + 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, 0xc25d9fc2U, 0xd36ebdd3U, + 0xacef43acU, 0x62a6c462U, 0x91a83991U, 0x95a43195U, 0xe437d3e4U, + 0x798bf279U, 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, 0x6cb4d86cU, + 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, 0x65afca65U, 0x7a8ef47aU, + 0xaee947aeU, 0x08181008U, 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, + 0x2e725c2eU, 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, 0x4bdd964bU, + 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, 0x7090e070U, 0x3e427c3eU, + 0xb5c471b5U, 0x66aacc66U, 0x48d89048U, 0x03050603U, 0xf601f7f6U, + 0x0e121c0eU, 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, 0xe138d9e1U, + 0xf813ebf8U, 0x98b32b98U, 0x11332211U, 0x69bbd269U, 0xd970a9d9U, + 0x8e89078eU, 0x94a73394U, 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, + 0xe920c9e9U, 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, 0xbfda65bfU, + 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, 0x41c38241U, 0x99b02999U, + 0x2d775a2dU, 0x0f111e0fU, 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, + 0x163a2c16U, +}; +static const u32 Te3[256] = { + + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, 0xf2f20dffU, + 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, 0x30305060U, 0x01010302U, + 0x6767a9ceU, 0x2b2b7d56U, 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, + 0x76769aecU, 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, 0xadadec41U, + 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, 0x9c9cbf23U, 0xa4a4f753U, + 0x727296e4U, 0xc0c05b9bU, 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, + 0x26266a4cU, 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, 0x717193e2U, + 0xd8d873abU, 0x31315362U, 0x15153f2aU, 0x04040c08U, 0xc7c75295U, + 0x23236546U, 0xc3c35e9dU, 0x18182830U, 0x9696a137U, 0x05050f0aU, + 0x9a9ab52fU, 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, 0x09091b12U, + 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, 0x1b1b2d36U, 0x6e6eb2dcU, + 0x5a5aeeb4U, 0xa0a0fb5bU, 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, + 0xb3b3ce7dU, 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, 0x20206040U, + 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, 0x6a6abed4U, 0xcbcb468dU, + 0xbebed967U, 0x39394b72U, 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, + 0xcfcf4a85U, 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, 0x4545cf8aU, + 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, 0x5050f0a0U, 0x3c3c4478U, + 0x9f9fba25U, 0xa8a8e34bU, 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, + 0x8f8f8a05U, 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, 0x10103020U, + 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, 0xcdcd4c81U, 0x0c0c1418U, + 0x13133526U, 0xecec2fc3U, 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, + 0x1717392eU, 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, 0x6060a0c0U, + 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, 0x22226644U, 0x2a2a7e54U, + 0x9090ab3bU, 0x8888830bU, 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, + 0x14143c28U, 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, 0x4949db92U, + 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, 0xc2c25d9fU, 0xd3d36ebdU, + 0xacacef43U, 0x6262a6c4U, 0x9191a839U, 0x9595a431U, 0xe4e437d3U, + 0x79798bf2U, 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, 0x6c6cb4d8U, + 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, 0x6565afcaU, 0x7a7a8ef4U, + 0xaeaee947U, 0x08081810U, 0xbabad56fU, 0x787888f0U, 0x25256f4aU, + 0x2e2e725cU, 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, 0x4b4bdd96U, + 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, 0x707090e0U, 0x3e3e427cU, + 0xb5b5c471U, 0x6666aaccU, 0x4848d890U, 0x03030506U, 0xf6f601f7U, + 0x0e0e121cU, 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, 0xe1e138d9U, + 0xf8f813ebU, 0x9898b32bU, 0x11113322U, 0x6969bbd2U, 0xd9d970a9U, + 0x8e8e8907U, 0x9494a733U, 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, + 0xe9e920c9U, 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, 0xbfbfda65U, + 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, 0x4141c382U, 0x9999b029U, + 0x2d2d775aU, 0x0f0f111eU, 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, + 0x16163a2cU, +}; +static const u32 Te4[256] = { + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, 0xf2f2f2f2U, + 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, 0x30303030U, 0x01010101U, + 0x67676767U, 0x2b2b2b2bU, 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, + 0x76767676U, 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, 0xadadadadU, + 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, 0x9c9c9c9cU, 0xa4a4a4a4U, + 0x72727272U, 0xc0c0c0c0U, 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, + 0x26262626U, 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, 0x71717171U, + 0xd8d8d8d8U, 0x31313131U, 0x15151515U, 0x04040404U, 0xc7c7c7c7U, + 0x23232323U, 0xc3c3c3c3U, 0x18181818U, 0x96969696U, 0x05050505U, + 0x9a9a9a9aU, 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, 0x09090909U, + 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, 0x1b1b1b1bU, 0x6e6e6e6eU, + 0x5a5a5a5aU, 0xa0a0a0a0U, 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, + 0xb3b3b3b3U, 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, 0x20202020U, + 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, 0x6a6a6a6aU, 0xcbcbcbcbU, + 0xbebebebeU, 0x39393939U, 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, + 0xcfcfcfcfU, 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, 0x45454545U, + 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, 0x50505050U, 0x3c3c3c3cU, + 0x9f9f9f9fU, 0xa8a8a8a8U, 0x51515151U, 0xa3a3a3a3U, 0x40404040U, + 0x8f8f8f8fU, 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, 0x10101010U, + 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, 0xcdcdcdcdU, 0x0c0c0c0cU, + 0x13131313U, 0xececececU, 0x5f5f5f5fU, 0x97979797U, 0x44444444U, + 0x17171717U, 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, 0x60606060U, + 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, 0x22222222U, 0x2a2a2a2aU, + 0x90909090U, 0x88888888U, 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, + 0x14141414U, 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, 0x49494949U, + 0x06060606U, 0x24242424U, 0x5c5c5c5cU, 0xc2c2c2c2U, 0xd3d3d3d3U, + 0xacacacacU, 0x62626262U, 0x91919191U, 0x95959595U, 0xe4e4e4e4U, + 0x79797979U, 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, 0x6c6c6c6cU, + 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, 0x65656565U, 0x7a7a7a7aU, + 0xaeaeaeaeU, 0x08080808U, 0xbabababaU, 0x78787878U, 0x25252525U, + 0x2e2e2e2eU, 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, 0x4b4b4b4bU, + 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, 0x70707070U, 0x3e3e3e3eU, + 0xb5b5b5b5U, 0x66666666U, 0x48484848U, 0x03030303U, 0xf6f6f6f6U, + 0x0e0e0e0eU, 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, 0xe1e1e1e1U, + 0xf8f8f8f8U, 0x98989898U, 0x11111111U, 0x69696969U, 0xd9d9d9d9U, + 0x8e8e8e8eU, 0x94949494U, 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, + 0xe9e9e9e9U, 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, 0xbfbfbfbfU, + 0xe6e6e6e6U, 0x42424242U, 0x68686868U, 0x41414141U, 0x99999999U, + 0x2d2d2d2dU, 0x0f0f0f0fU, 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, + 0x16161616U, +}; +static const u32 Td0[256] = { + 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, 0x3bab6bcbU, + 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, 0x2030fa55U, 0xad766df6U, + 0x88cc7691U, 0xf5024c25U, 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, + 0xb562a38fU, 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U, + 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U, 0x038f5fe7U, + 0x15929c95U, 0xbf6d7aebU, 0x955259daU, 0xd4be832dU, 0x587421d3U, + 0x49e06929U, 0x8ec9c844U, 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, + 0x27b971ddU, 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U, + 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U, 0xb16477e0U, + 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U, 0x70486858U, 0x8f45fd19U, + 0x94de6c87U, 0x527bf8b7U, 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, + 0x6655ab2aU, 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U, + 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU, 0x8acf1c2bU, + 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U, 0x65daf4cdU, 0x0605bed5U, + 0xd134621fU, 0xc4a6fe8aU, 0x342e539dU, 0xa2f355a0U, 0x058ae132U, + 0xa4f6eb75U, 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U, + 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U, 0x91548db5U, + 0x71c45d05U, 0x0406d46fU, 0x605015ffU, 0x1998fb24U, 0xd6bde997U, + 0x894043ccU, 0x67d99e77U, 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, + 0x79c8eedbU, 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U, + 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU, 0xfd0efffbU, + 0x0f853856U, 0x3daed51eU, 0x362d3927U, 0x0a0fd964U, 0x685ca621U, + 0x9b5b54d1U, 0x24362e3aU, 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, + 0x1b9b919eU, 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U, + 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU, 0x0e090d0bU, + 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U, 0x57f11985U, 0xaf75074cU, + 0xee99ddbbU, 0xa37f60fdU, 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, + 0x5bfb7e34U, 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U, + 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U, 0x854a247dU, + 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU, 0x1d9e2f4bU, 0xdcb230f3U, + 0x0d8652ecU, 0x77c1e3d0U, 0x2bb3166cU, 0xa970b999U, 0x119448faU, + 0x47e96422U, 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU, + 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U, 0xa6f581cfU, + 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U, 0x2c3a9de4U, 0x5078920dU, + 0x6a5fcc9bU, 0x547e4662U, 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, + 0x82c3aff5U, 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U, + 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU, 0xcd267809U, + 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U, 0xe6956e65U, 0xaaffe67eU, + 0x21bccf08U, 0xef15e8e6U, 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, + 0x29b07cd6U, 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U, + 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U, 0xf104984aU, + 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU, 0x764dd68dU, 0x43efb04dU, + 0xccaa4d54U, 0xe49604dfU, 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, + 0x4665517fU, 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU, + 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U, 0x9ad7618cU, + 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U, 0xcea927eeU, 0xb761c935U, + 0xe11ce5edU, 0x7a47b13cU, 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, + 0x73c737bfU, 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U, + 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU, 0x161dc372U, + 0xbce2250cU, 0x283c498bU, 0xff0d9541U, 0x39a80171U, 0x080cb3deU, + 0xd8b4e49cU, 0x6456c190U, 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, + 0xd0b85742U, +}; +static const u32 Td1[256] = { + 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU, 0xcb3bab6bU, + 0xf11f9d45U, 0xabacfa58U, 0x934be303U, 0x552030faU, 0xf6ad766dU, + 0x9188cc76U, 0x25f5024cU, 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, + 0x8fb562a3U, 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U, + 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U, 0xe7038f5fU, + 0x9515929cU, 0xebbf6d7aU, 0xda955259U, 0x2dd4be83U, 0xd3587421U, + 0x2949e069U, 0x448ec9c8U, 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, + 0xdd27b971U, 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU, + 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU, 0xe0b16477U, + 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU, 0x58704868U, 0x198f45fdU, + 0x8794de6cU, 0xb7527bf8U, 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, + 0x2a6655abU, 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U, + 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U, 0x2b8acf1cU, + 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U, 0xcd65daf4U, 0xd50605beU, + 0x1fd13462U, 0x8ac4a6feU, 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, + 0x75a4f6ebU, 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U, + 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU, 0xb591548dU, + 0x0571c45dU, 0x6f0406d4U, 0xff605015U, 0x241998fbU, 0x97d6bde9U, + 0xcc894043U, 0x7767d99eU, 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, + 0xdb79c8eeU, 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U, + 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U, 0xfbfd0effU, + 0x560f8538U, 0x1e3daed5U, 0x27362d39U, 0x640a0fd9U, 0x21685ca6U, + 0xd19b5b54U, 0x3a24362eU, 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, + 0x9e1b9b91U, 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU, + 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U, 0x0b0e090dU, + 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U, 0x8557f119U, 0x4caf7507U, + 0xbbee99ddU, 0xfda37f60U, 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, + 0x345bfb7eU, 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U, + 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U, 0x7d854a24U, + 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U, 0x4b1d9e2fU, 0xf3dcb230U, + 0xec0d8652U, 0xd077c1e3U, 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, + 0x2247e964U, 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U, + 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU, 0xcfa6f581U, + 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU, 0xe42c3a9dU, 0x0d507892U, + 0x9b6a5fccU, 0x62547e46U, 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, + 0xf582c3afU, 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U, + 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU, 0x09cd2678U, + 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU, 0x65e6956eU, 0x7eaaffe6U, + 0x0821bccfU, 0xe6ef15e8U, 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, + 0xd629b07cU, 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U, + 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U, 0x4af10498U, + 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U, 0x8d764dd6U, 0x4d43efb0U, + 0x54ccaa4dU, 0xdfe49604U, 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, + 0x7f466551U, 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U, + 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U, 0x8c9ad761U, + 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU, 0xeecea927U, 0x35b761c9U, + 0xede11ce5U, 0x3c7a47b1U, 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, + 0xbf73c737U, 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU, + 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U, 0x72161dc3U, + 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U, 0x7139a801U, 0xde080cb3U, + 0x9cd8b4e4U, 0x906456c1U, 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, + 0x42d0b857U, +}; +static const u32 Td2[256] = { + 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U, 0x6bcb3babU, + 0x45f11f9dU, 0x58abacfaU, 0x03934be3U, 0xfa552030U, 0x6df6ad76U, + 0x769188ccU, 0x4c25f502U, 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, + 0xa38fb562U, 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU, + 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U, 0x5fe7038fU, + 0x9c951592U, 0x7aebbf6dU, 0x59da9552U, 0x832dd4beU, 0x21d35874U, + 0x692949e0U, 0xc8448ec9U, 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, + 0x71dd27b9U, 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU, + 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U, 0x77e0b164U, + 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U, 0x68587048U, 0xfd198f45U, + 0x6c8794deU, 0xf8b7527bU, 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, + 0xab2a6655U, 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U, + 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U, 0x1c2b8acfU, + 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U, 0xf4cd65daU, 0xbed50605U, + 0x621fd134U, 0xfe8ac4a6U, 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, + 0xeb75a4f6U, 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU, + + 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U, 0x8db59154U, + 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U, 0xfb241998U, 0xe997d6bdU, + 0x43cc8940U, 0x9e7767d9U, 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, + 0xeedb79c8U, 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U, + 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU, 0xfffbfd0eU, + 0x38560f85U, 0xd51e3daeU, 0x3927362dU, 0xd9640a0fU, 0xa621685cU, + 0x54d19b5bU, 0x2e3a2436U, 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, + 0x919e1b9bU, 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U, + 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU, 0x0d0b0e09U, + 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU, 0x198557f1U, 0x074caf75U, + 0xddbbee99U, 0x60fda37fU, 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, + 0x7e345bfbU, 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U, + 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U, 0x247d854aU, + 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U, 0x2f4b1d9eU, 0x30f3dcb2U, + 0x52ec0d86U, 0xe3d077c1U, 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, + 0x642247e9U, 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U, + 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U, 0x81cfa6f5U, + 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU, 0x9de42c3aU, 0x920d5078U, + 0xcc9b6a5fU, 0x4662547eU, 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, + 0xaff582c3U, 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U, + 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU, 0x7809cd26U, + 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU, 0x6e65e695U, 0xe67eaaffU, + 0xcf0821bcU, 0xe8e6ef15U, 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, + 0x7cd629b0U, 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U, + 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U, 0x984af104U, + 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U, 0xd68d764dU, 0xb04d43efU, + 0x4d54ccaaU, 0x04dfe496U, 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, + 0x517f4665U, 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU, + 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U, 0x618c9ad7U, + 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U, 0x27eecea9U, 0xc935b761U, + 0xe5ede11cU, 0xb13c7a47U, 0xdf599cd2U, 0x733f55f2U, 0xce791814U, + 0x37bf73c7U, 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U, + 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U, 0xc372161dU, + 0x250cbce2U, 0x498b283cU, 0x9541ff0dU, 0x017139a8U, 0xb3de080cU, + 0xe49cd8b4U, 0xc1906456U, 0x84617bcbU, 0xb670d532U, 0x5c74486cU, + 0x5742d0b8U, +}; +static const u32 Td3[256] = { + 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU, 0xab6bcb3bU, + 0x9d45f11fU, 0xfa58abacU, 0xe303934bU, 0x30fa5520U, 0x766df6adU, + 0xcc769188U, 0x024c25f5U, 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, + 0x62a38fb5U, 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU, + 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU, 0x8f5fe703U, + 0x929c9515U, 0x6d7aebbfU, 0x5259da95U, 0xbe832dd4U, 0x7421d358U, + 0xe0692949U, 0xc9c8448eU, 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, + 0xb971dd27U, 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU, + 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U, 0x6477e0b1U, + 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U, 0x48685870U, 0x45fd198fU, + 0xde6c8794U, 0x7bf8b752U, 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, + 0x55ab2a66U, 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U, + 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU, 0xcf1c2b8aU, + 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU, 0xdaf4cd65U, 0x05bed506U, + 0x34621fd1U, 0xa6fe8ac4U, 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, + 0xf6eb75a4U, 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU, + 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU, 0x548db591U, + 0xc45d0571U, 0x06d46f04U, 0x5015ff60U, 0x98fb2419U, 0xbde997d6U, + 0x4043cc89U, 0xd99e7767U, 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, + 0xc8eedb79U, 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U, + 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU, 0x0efffbfdU, + 0x8538560fU, 0xaed51e3dU, 0x2d392736U, 0x0fd9640aU, 0x5ca62168U, + 0x5b54d19bU, 0x362e3a24U, 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, + 0x9b919e1bU, 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU, + 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U, 0x090d0b0eU, + 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U, 0xf1198557U, 0x75074cafU, + 0x99ddbbeeU, 0x7f60fda3U, 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, + 0xfb7e345bU, 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U, + 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U, 0x4a247d85U, + 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U, 0x9e2f4b1dU, 0xb230f3dcU, + 0x8652ec0dU, 0xc1e3d077U, 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, + 0xe9642247U, 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U, + 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U, 0xf581cfa6U, + 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU, 0x3a9de42cU, 0x78920d50U, + 0x5fcc9b6aU, 0x7e466254U, 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, + 0xc3aff582U, 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU, + 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU, 0x267809cdU, + 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U, 0x956e65e6U, 0xffe67eaaU, + 0xbccf0821U, 0x15e8e6efU, 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, + 0xb07cd629U, 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U, + 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U, 0x04984af1U, + 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U, 0x4dd68d76U, 0xefb04d43U, + 0xaa4d54ccU, 0x9604dfe4U, 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, + 0x65517f46U, 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU, + 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU, 0xd7618c9aU, + 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU, 0xa927eeceU, 0x61c935b7U, + 0x1ce5ede1U, 0x47b13c7aU, 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, + 0xc737bf73U, 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U, + 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U, 0x1dc37216U, + 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, 0xa8017139U, 0x0cb3de08U, + 0xb4e49cd8U, 0x56c19064U, 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, + 0xb85742d0U, +}; +static const u32 Td4[256] = { + 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, 0x30303030U, + 0x36363636U, 0xa5a5a5a5U, 0x38383838U, 0xbfbfbfbfU, 0x40404040U, + 0xa3a3a3a3U, 0x9e9e9e9eU, 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, + 0xfbfbfbfbU, 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, + 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, 0x34343434U, + 0x8e8e8e8eU, 0x43434343U, 0x44444444U, 0xc4c4c4c4U, 0xdedededeU, + 0xe9e9e9e9U, 0xcbcbcbcbU, 0x54545454U, 0x7b7b7b7bU, 0x94949494U, + 0x32323232U, 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, + 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, 0x42424242U, + 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, 0x08080808U, 0x2e2e2e2eU, + 0xa1a1a1a1U, 0x66666666U, 0x28282828U, 0xd9d9d9d9U, 0x24242424U, + 0xb2b2b2b2U, 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, + 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, 0x72727272U, + 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, 0x86868686U, 0x68686868U, + 0x98989898U, 0x16161616U, 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, + 0xccccccccU, 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, + 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, 0xfdfdfdfdU, + 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, 0x5e5e5e5eU, 0x15151515U, + 0x46464646U, 0x57575757U, 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, + 0x84848484U, 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, + 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, 0xf7f7f7f7U, + 0xe4e4e4e4U, 0x58585858U, 0x05050505U, 0xb8b8b8b8U, 0xb3b3b3b3U, + 0x45454545U, 0x06060606U, 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, + 0x8f8f8f8fU, 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, + 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, 0x01010101U, + 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, 0x3a3a3a3aU, 0x91919191U, + 0x11111111U, 0x41414141U, 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, + 0xeaeaeaeaU, 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, + 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, 0x96969696U, + 0xacacacacU, 0x74747474U, 0x22222222U, 0xe7e7e7e7U, 0xadadadadU, + 0x35353535U, 0x85858585U, 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, + 0xe8e8e8e8U, 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, + 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, 0x1d1d1d1dU, + 0x29292929U, 0xc5c5c5c5U, 0x89898989U, 0x6f6f6f6fU, 0xb7b7b7b7U, + 0x62626262U, 0x0e0e0e0eU, 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, + 0x1b1b1b1bU, 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, + 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, 0x9a9a9a9aU, + 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, 0x78787878U, 0xcdcdcdcdU, + 0x5a5a5a5aU, 0xf4f4f4f4U, 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, + 0x33333333U, 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, + 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, 0x27272727U, + 0x80808080U, 0xececececU, 0x5f5f5f5fU, 0x60606060U, 0x51515151U, + 0x7f7f7f7fU, 0xa9a9a9a9U, 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, + 0x0d0d0d0dU, 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, + 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, 0xa0a0a0a0U, + 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, 0xaeaeaeaeU, 0x2a2a2a2aU, + 0xf5f5f5f5U, 0xb0b0b0b0U, 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, + 0x3c3c3c3cU, 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, + 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, 0xbabababaU, + 0x77777777U, 0xd6d6d6d6U, 0x26262626U, 0xe1e1e1e1U, 0x69696969U, + 0x14141414U, 0x63636363U, 0x55555555U, 0x21212121U, 0x0c0c0c0cU, + 0x7d7d7d7dU, +}; +static const u32 rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000, + 0x40000000, 0x80000000, 0x1B000000, 0x36000000, /* for 128-bit blocks, + Rijndael never uses + more than 10 rcon + values */ +}; + +#define SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00) + +#ifdef _MSC_VER +#define GETU32(p) SWAP(*((u32 *) (p))) +#define PUTU32(ct, st) \ + { *((u32 *) (ct)) = SWAP((st)); } +#else +#define GETU32(pt) \ + (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] << 8) ^ \ + ((u32)(pt)[3])) +#define PUTU32(ct, st) \ + { \ + (ct)[0] = (u8)((st) >> 24); \ + (ct)[1] = (u8)((st) >> 16); \ + (ct)[2] = (u8)((st) >> 8); \ + (ct)[3] = (u8)(st); \ + } +#endif + +/** + * Expand the cipher key into the encryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits) { + int i = 0; + u32 temp; + + rk[0] = GETU32(cipherKey); + rk[1] = GETU32(cipherKey + 4); + rk[2] = GETU32(cipherKey + 8); + rk[3] = GETU32(cipherKey + 12); + if (keyBits == 128) { + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp) &0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + return 10; + } + rk += 4; + } + } + rk[4] = GETU32(cipherKey + 16); + rk[5] = GETU32(cipherKey + 20); + if (keyBits == 192) { + for (;;) { + temp = rk[5]; + rk[6] = rk[0] ^ (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp) &0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[7] = rk[1] ^ rk[6]; + rk[8] = rk[2] ^ rk[7]; + rk[9] = rk[3] ^ rk[8]; + if (++i == 8) { + return 12; + } + rk[10] = rk[4] ^ rk[9]; + rk[11] = rk[5] ^ rk[10]; + rk += 6; + } + } + rk[6] = GETU32(cipherKey + 24); + rk[7] = GETU32(cipherKey + 28); + if (keyBits == 256) { + for (;;) { + temp = rk[7]; + rk[8] = rk[0] ^ (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp) &0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24)] & 0x000000ff) ^ rcon[i]; + rk[9] = rk[1] ^ rk[8]; + rk[10] = rk[2] ^ rk[9]; + rk[11] = rk[3] ^ rk[10]; + if (++i == 7) { + return 14; + } + temp = rk[11]; + rk[12] = rk[4] ^ (Te4[(temp >> 24)] & 0xff000000) ^ + (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(temp) &0xff] & 0x000000ff); + rk[13] = rk[5] ^ rk[12]; + rk[14] = rk[6] ^ rk[13]; + rk[15] = rk[7] ^ rk[14]; + + rk += 8; + } + } + return 0; +} + +/** + * Expand the cipher key into the decryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits) { + int Nr, i, j; + u32 temp; + + /* expand the cipher key: */ + Nr = rijndaelKeySetupEnc(rk, cipherKey, keyBits); + /* invert the order of the round keys: */ + for (i = 0, j = 4 * Nr; i < j; i += 4, j -= 4) { + temp = rk[i]; + rk[i] = rk[j]; + rk[j] = temp; + temp = rk[i + 1]; + rk[i + 1] = rk[j + 1]; + rk[j + 1] = temp; + temp = rk[i + 2]; + rk[i + 2] = rk[j + 2]; + rk[j + 2] = temp; + temp = rk[i + 3]; + rk[i + 3] = rk[j + 3]; + rk[j + 3] = temp; + } + /* apply the inverse MixColumn transform to all round keys but the first + * and the last: */ + for (i = 1; i < Nr; i++) { + rk += 4; + rk[0] = Td0[Te4[(rk[0] >> 24)] & 0xff] ^ + Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[0]) & 0xff] & 0xff]; + rk[1] = Td0[Te4[(rk[1] >> 24)] & 0xff] ^ + Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[1]) & 0xff] & 0xff]; + rk[2] = Td0[Te4[(rk[2] >> 24)] & 0xff] ^ + Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[2]) & 0xff] & 0xff]; + rk[3] = Td0[Te4[(rk[3] >> 24)] & 0xff] ^ + Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[(rk[3]) & 0xff] & 0xff]; + } + return Nr; +} + +void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], + u8 ct[16]) { + u32 s0, s1, s2, s3, t0, t1, t2, t3; +#ifndef FULL_UNROLL + int r; +#endif /* ?FULL_UNROLL */ + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(pt) ^ rk[0]; + s1 = GETU32(pt + 4) ^ rk[1]; + s2 = GETU32(pt + 8) ^ rk[2]; + s3 = GETU32(pt + 12) ^ rk[3]; +#ifdef FULL_UNROLL + /* round 1: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[4]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[5]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[6]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[7]; + /* round 2: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[8]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[9]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[10]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[12]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[13]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[14]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[16]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[17]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[18]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[20]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[21]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[22]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[24]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[25]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[26]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[28]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[29]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[30]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[32]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[33]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[34]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[36]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[37]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[38]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[39]; + if (Nr > 10) { + /* round 10: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ rk[40]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ rk[41]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ rk[42]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ rk[43]; + /* round 11: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ rk[44]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ rk[45]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ rk[46]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ rk[47]; + if (Nr > 12) { + /* round 12: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[48]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[49]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[50]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[51]; + /* round 13: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[52]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[53]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[54]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[55]; + } + } + rk += Nr << 2; +#else /* !FULL_UNROLL */ + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = Te0[(s0 >> 24)] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[(s3) &0xff] ^ rk[4]; + t1 = Te0[(s1 >> 24)] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[(s0) &0xff] ^ rk[5]; + t2 = Te0[(s2 >> 24)] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[(s1) &0xff] ^ rk[6]; + t3 = Te0[(s3 >> 24)] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[(s2) &0xff] ^ rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = Te0[(t0 >> 24)] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ + Te3[(t3) &0xff] ^ rk[0]; + s1 = Te0[(t1 >> 24)] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ + Te3[(t0) &0xff] ^ rk[1]; + s2 = Te0[(t2 >> 24)] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ + Te3[(t1) &0xff] ^ rk[2]; + s3 = Te0[(t3 >> 24)] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ + Te3[(t2) &0xff] ^ rk[3]; + } +#endif /* ?FULL_UNROLL */ + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = (Te4[(t0 >> 24)] & 0xff000000) ^ + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ (Te4[(t3) &0xff] & 0x000000ff) ^ + rk[0]; + PUTU32(ct, s0); + s1 = (Te4[(t1 >> 24)] & 0xff000000) ^ + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ (Te4[(t0) &0xff] & 0x000000ff) ^ + rk[1]; + PUTU32(ct + 4, s1); + s2 = (Te4[(t2 >> 24)] & 0xff000000) ^ + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ (Te4[(t1) &0xff] & 0x000000ff) ^ + rk[2]; + PUTU32(ct + 8, s2); + s3 = (Te4[(t3 >> 24)] & 0xff000000) ^ + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ (Te4[(t2) &0xff] & 0x000000ff) ^ + rk[3]; + PUTU32(ct + 12, s3); +} + +void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], + u8 pt[16]) { + u32 s0, s1, s2, s3, t0, t1, t2, t3; +#ifndef FULL_UNROLL + int r; +#endif /* ?FULL_UNROLL */ + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(ct) ^ rk[0]; + s1 = GETU32(ct + 4) ^ rk[1]; + s2 = GETU32(ct + 8) ^ rk[2]; + s3 = GETU32(ct + 12) ^ rk[3]; +#ifdef FULL_UNROLL + /* round 1: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[4]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[5]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[6]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[7]; + /* round 2: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[8]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[9]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[10]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[12]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[13]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[14]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[16]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[17]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[18]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[20]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[21]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[22]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[24]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[25]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[26]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[28]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[29]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[30]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[32]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[33]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[34]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[36]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[37]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[38]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[39]; + if (Nr > 10) { + /* round 10: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ rk[40]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ rk[41]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ rk[42]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ rk[43]; + /* round 11: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ rk[44]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ rk[45]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ rk[46]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ rk[47]; + if (Nr > 12) { + /* round 12: */ + s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ Td3[t1 & 0xff] ^ rk[48]; + s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ Td3[t2 & 0xff] ^ rk[49]; + s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ Td3[t3 & 0xff] ^ rk[50]; + s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ Td3[t0 & 0xff] ^ rk[51]; + /* round 13: */ + t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ Td3[s1 & 0xff] ^ rk[52]; + t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ Td3[s2 & 0xff] ^ rk[53]; + t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ Td3[s3 & 0xff] ^ rk[54]; + t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ Td3[s0 & 0xff] ^ rk[55]; + } + } + rk += Nr << 2; +#else /* !FULL_UNROLL */ + /* + * Nr - 1 full rounds: + */ + r = Nr >> 1; + for (;;) { + t0 = Td0[(s0 >> 24)] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[(s1) &0xff] ^ rk[4]; + t1 = Td0[(s1 >> 24)] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[(s2) &0xff] ^ rk[5]; + t2 = Td0[(s2 >> 24)] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[(s3) &0xff] ^ rk[6]; + t3 = Td0[(s3 >> 24)] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[(s0) &0xff] ^ rk[7]; + + rk += 8; + if (--r == 0) { + break; + } + + s0 = Td0[(t0 >> 24)] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] ^ + Td3[(t1) &0xff] ^ rk[0]; + s1 = Td0[(t1 >> 24)] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] ^ + Td3[(t2) &0xff] ^ rk[1]; + s2 = Td0[(t2 >> 24)] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] ^ + Td3[(t3) &0xff] ^ rk[2]; + s3 = Td0[(t3 >> 24)] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] ^ + Td3[(t0) &0xff] ^ rk[3]; + } +#endif /* ?FULL_UNROLL */ + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = (Td4[(t0 >> 24)] & 0xff000000) ^ + (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(t1) &0xff] & 0x000000ff) ^ + rk[0]; + PUTU32(pt, s0); + s1 = (Td4[(t1 >> 24)] & 0xff000000) ^ + (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(t2) &0xff] & 0x000000ff) ^ + rk[1]; + PUTU32(pt + 4, s1); + s2 = (Td4[(t2 >> 24)] & 0xff000000) ^ + (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(t3) &0xff] & 0x000000ff) ^ + rk[2]; + PUTU32(pt + 8, s2); + s3 = (Td4[(t3 >> 24)] & 0xff000000) ^ + (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(t0) &0xff] & 0x000000ff) ^ + rk[3]; + PUTU32(pt + 12, s3); +} + +#ifdef INTERMEDIATE_VALUE_KAT + +void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds) { + int r; + u32 s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(block) ^ rk[0]; + s1 = GETU32(block + 4) ^ rk[1]; + s2 = GETU32(block + 8) ^ rk[2]; + s3 = GETU32(block + 12) ^ rk[3]; + rk += 4; + + /* + * Nr - 1 full rounds: + */ + for (r = (rounds < Nr ? rounds : Nr - 1); r > 0; r--) { + t0 = Te0[(s0 >> 24)] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ + Te3[(s3) &0xff] ^ rk[0]; + t1 = Te0[(s1 >> 24)] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ + Te3[(s0) &0xff] ^ rk[1]; + t2 = Te0[(s2 >> 24)] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ + Te3[(s1) &0xff] ^ rk[2]; + t3 = Te0[(s3 >> 24)] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ + Te3[(s2) &0xff] ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + rk += 4; + } + + /* + * apply last round and + * map cipher state to byte array block: + */ + if (rounds == Nr) { + t0 = (Te4[(s0 >> 24)] & 0xff000000) ^ + (Te4[(s1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s3) &0xff] & 0x000000ff) ^ rk[0]; + t1 = (Te4[(s1 >> 24)] & 0xff000000) ^ + (Te4[(s2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s0) &0xff] & 0x000000ff) ^ rk[1]; + t2 = (Te4[(s2 >> 24)] & 0xff000000) ^ + (Te4[(s3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s1) &0xff] & 0x000000ff) ^ rk[2]; + t3 = (Te4[(s3 >> 24)] & 0xff000000) ^ + (Te4[(s0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(s1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(s2) &0xff] & 0x000000ff) ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + } + + PUTU32(block, s0); + PUTU32(block + 4, s1); + PUTU32(block + 8, s2); + PUTU32(block + 12, s3); +} + +void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds) { + int r; + u32 s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(block) ^ rk[0]; + s1 = GETU32(block + 4) ^ rk[1]; + s2 = GETU32(block + 8) ^ rk[2]; + s3 = GETU32(block + 12) ^ rk[3]; + rk += 4; + + /* + * Nr - 1 full rounds: + */ + for (r = (rounds < Nr ? rounds : Nr) - 1; r > 0; r--) { + t0 = Td0[(s0 >> 24)] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] ^ + Td3[(s1) &0xff] ^ rk[0]; + t1 = Td0[(s1 >> 24)] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] ^ + Td3[(s2) &0xff] ^ rk[1]; + t2 = Td0[(s2 >> 24)] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] ^ + Td3[(s3) &0xff] ^ rk[2]; + t3 = Td0[(s3 >> 24)] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] ^ + Td3[(s0) &0xff] ^ rk[3]; + + s0 = t0; + s1 = t1; + s2 = t2; + s3 = t3; + rk += 4; + } + + /* + * complete the last round and + * map cipher state to byte array block: + */ + t0 = (Td4[(s0 >> 24)] & 0xff000000) ^ + (Td4[(s3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s2 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(s1) &0xff] & 0x000000ff); + t1 = (Td4[(s1 >> 24)] & 0xff000000) ^ + (Td4[(s0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s3 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(s2) &0xff] & 0x000000ff); + t2 = (Td4[(s2 >> 24)] & 0xff000000) ^ + (Td4[(s1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s0 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(s3) &0xff] & 0x000000ff); + t3 = (Td4[(s3 >> 24)] & 0xff000000) ^ + (Td4[(s2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(s1 >> 8) & 0xff] & 0x0000ff00) ^ (Td4[(s0) &0xff] & 0x000000ff); + + if (rounds == Nr) { + t0 ^= rk[0]; + t1 ^= rk[1]; + t2 ^= rk[2]; + t3 ^= rk[3]; + } + + PUTU32(block, t0); + PUTU32(block + 4, t1); + PUTU32(block + 8, t2); + PUTU32(block + 12, t3); +} + +#endif /* INTERMEDIATE_VALUE_KAT */ diff --git a/lib/rijndael-alg-fst.h b/lib/rijndael-alg-fst.h new file mode 100644 index 0000000..830f16a --- /dev/null +++ b/lib/rijndael-alg-fst.h @@ -0,0 +1,57 @@ +/** + * rijndael-alg-fst.h + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RIJNDAEL_ALG_FST_H +#define RIJNDAEL_ALG_FST_H + +#define MAXKC (256 / 32) +#define MAXKB (256 / 8) +#define MAXNR 14 + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; + +int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits); + +int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], + int keyBits); + +void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], + u8 ct[16]); + +void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], + u8 pt[16]); + +#ifdef INTERMEDIATE_VALUE_KAT +void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds); +void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], + int rounds); +#endif /* INTERMEDIATE_VALUE_KAT */ + +#endif /* RIJNDAEL_ALG_FST_H */ diff --git a/lib/types.h b/lib/types.h new file mode 100644 index 0000000..6c50af3 --- /dev/null +++ b/lib/types.h @@ -0,0 +1,19 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_TYPES_H +#define XMAP_TYPES_H + +#include + +typedef uint8_t ipaddr_n_t; // IPv4/6 address network order +typedef uint16_t port_h_t; // port host order +typedef unsigned char macaddr_t; + +#endif // XMAP_TYPES_H diff --git a/lib/util.c b/lib/util.c new file mode 100644 index 0000000..929daa8 --- /dev/null +++ b/lib/util.c @@ -0,0 +1,454 @@ +#define _GNU_SOURCE + +#include "util.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "includes.h" +#include "logger.h" +#include "xalloc.h" + +#define MAX_SPLITS 128 + +int max_int(int a, int b) { + if (a >= b) { + return a; + } + + return b; +} + +void enforce_range(const char *name, int v, int min, int max) { + if (check_range(v, min, max) == EXIT_FAILURE) { + log_fatal("xmap", "argument `%s' must be between %d and %d\n", name, + min, max); + } +} + +void split_string(char *in, int *len, char ***results) { + char **fields = xcalloc(MAX_SPLITS, sizeof(char *)); + int retvlen = 0; + char * currloc = in; + + // parse csv into a set of strings + while (1) { + assert(retvlen < MAX_SPLITS); + size_t len = strcspn(currloc, ", "); + if (len == 0) { + currloc++; + } else { + char *new = xmalloc(len + 1); + strncpy(new, currloc, len); + new[len] = '\0'; + fields[retvlen++] = new; + assert(fields[retvlen - 1]); + } + if (len == strlen(currloc)) { + break; + } + currloc += len; + } + + *results = fields; + *len = retvlen; +} + +void fprintw(FILE *f, char *s, size_t w) { + if (strlen(s) <= w) { + fprintf(f, "%s", s); + return; + } + + // process each line individually in order to + // respect existing line breaks in string. + char *news = strdup(s); + char *pch = strtok(news, "\n"); + while (pch) { + if (strlen(pch) <= w) { + printf("%s\n", pch); + pch = strtok(NULL, "\n"); + continue; + } + char *t = pch; + while (strlen(t)) { + size_t numchars = 0; // number of chars to print + char * tmp = t; + while (1) { + size_t new = strcspn(tmp, " ") + 1; + if (new == strlen(tmp) || new > w) { + // there are no spaces in the string, + // so, just print the entire thing on + // one line; + numchars += new; + break; + } else if (numchars + new > w) { + // if we added any more, we'd be over w + // chars so time to print the line and + // move on to the next. + break; + } else { + tmp += (size_t) new; + numchars += new; + } + } + fprintf(f, "%.*s\n", (int) numchars, t); + t += (size_t) numchars; + if (t > pch + (size_t) strlen(pch)) { + break; + } + } + pch = strtok(NULL, "\n"); + } + + free(news); +} + +uint32_t parse_max_hosts(char *max_targets) { + int errno = 0; + char * end; + double v = strtod(max_targets, &end); + if (end == max_targets || errno != 0) { + log_fatal("argparse", "can't convert max-targets to a number"); + } + + if (end[0] == '%' && end[1] == '\0') { + // treat as percentage + v = v * ((unsigned long long int) 1 << 32u) / 100.; + } else if (end[0] != '\0') { + log_fatal("eargparse", "extra characters after max-targets"); + } + + if (v <= 0) { + return 0; + } else if (v >= ((unsigned long long int) 1 << 32u)) { + return 0xFFFFFFFF; + } else { + return v; + } +} + +// pretty print elapsed (or estimated) number of seconds +void time_string(uint32_t time, int est, char *buf, size_t len) { + int y = time / 31556736; + int d = (time % 31556736) / 86400; + int h = (time % 86400) / 3600; + int m = (time % 3600) / 60; + int s = time % 60; + + if (est) { + if (y > 0) { + snprintf(buf, len, "%d years", y); + } else if (d > 9) { + snprintf(buf, len, "%dd", d); + } else if (d > 0) { + snprintf(buf, len, "%dd%02dh", d, h); + } else if (h > 9) { + snprintf(buf, len, "%dh", h); + } else if (h > 0) { + snprintf(buf, len, "%dh%02dm", h, m); + } else if (m > 9) { + snprintf(buf, len, "%dm", m); + } else if (m > 0) { + snprintf(buf, len, "%dm%02ds", m, s); + } else { + snprintf(buf, len, "%ds", s); + } + } else { + if (d > 0) { + snprintf(buf, len, "%dd%d:%02d:%02d", d, h, m, s); + } else if (h > 0) { + snprintf(buf, len, "%d:%02d:%02d", h, m, s); + } else { + snprintf(buf, len, "%d:%02d", m, s); + } + } +} + +// pretty print quantities +void number_string(uint64_t n, char *buf, size_t len) { + int figs = 0; + if (n < 1e3) { + snprintf(buf, len, "%u ", n); + } else if (n < 1e6) { + if (n < 1e4) { + figs = 2; + } else if (n < 1e5) { + figs = 1; + } + snprintf(buf, len, "%0.*f K", figs, (float) n / (float) 1e3); + } else if (n < 1e9) { + if (n < 1e7) { + figs = 2; + } else if (n < 1e8) { + figs = 1; + } + snprintf(buf, len, "%0.*f M", figs, (float) n / (float) 1e6); + } else { // 1e12 + if (n < 1e10) { + figs = 2; + } else if (n < 1e11) { + figs = 1; + } + snprintf(buf, len, "%0.*f G", figs, (float) n / (float) 1e9); + } +} + +// pretty print quantities (bandwidth) +void bits_string(uint64_t n, char *buf, size_t len) { + int figs = 0; + if (n < 1024) { + snprintf(buf, len, "%u ", n); + } else if (n < 1048576) { // 1024^2 + if (n < 65536) { // /16 + figs = 2; + } else if (n < 262144) { // /4 + figs = 1; + } + snprintf(buf, len, "%0.*f K", figs, (float) n / (float) 1024); // 1024 + } else if (n < 1073741824) { // 1024^3 + if (n < 67108864) { // 16 + figs = 2; + } else if (n < 268435456) { // /4 + figs = 1; + } + snprintf(buf, len, "%0.*f M", figs, + (float) n / (float) 1048576); // 1024^2 + } else { // 1024^4 + if (n < 68719476736) { // /16 + figs = 2; + } else if (n < 274877906944) { // /4 + figs = 1; + } + snprintf(buf, len, "%0.*f G", figs, + (float) n / (float) 1073741824); // 1024^3 + } +} + +int parse_mac(macaddr_t *out, char *in) { + if (strlen(in) < MAC_ADDR_LEN * 3 - 1) return 0; + char octet[4]; + octet[2] = '\0'; + + for (int i = 0; i < MAC_ADDR_LEN; i++) { + if (i < MAC_ADDR_LEN - 1 && in[i * 3 + 2] != ':') { + return 0; + } + strncpy(octet, &in[i * 3], 2); + char *err = NULL; + long b = strtol(octet, &err, 16); + if (err && *err != '\0') { + return 0; + } + out[i] = b & 0xFF; + } + + return 1; +} + +int check_range(int v, int min, int max) { + if (v < min || v > max) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int file_exists(char *name) { + FILE *file = fopen(name, "r"); + if (!file) return 0; + fclose(file); + + return 1; +} + +// IP string to IP address struct +int inet_str2in(const char *str, void *in, int ipv46_flag) { + switch (ipv46_flag) { + case IPV6_FLAG: { + int ret = inet_pton(AF_INET6, str, (struct in6_addr *) in); + if (ret != 1) { + return 0; + } + return 1; + } + case IPV4_FLAG: { + int ret = inet_pton(AF_INET, str, (struct in_addr *) in); + if (ret != 1) { + return 0; + } + return 1; + } + default: + return 0; + } +} + +// IP address struct to IP string +int inet_in2str(const void *in, char *str, int len, int ipv46_flag) { + switch (ipv46_flag) { + case IPV6_FLAG: { + char *err = (char *) inet_ntop(AF_INET6, in, str, len); + if (err == NULL) { + return 0; + } + return 1; + } + case IPV4_FLAG: { + char *err = (char *) inet_ntop(AF_INET, in, str, len); + if (err == NULL) { + return 0; + } + return 1; + } + default: + return 0; + } +} + +// Note: caller must free return value +char *inet_in2constr(const void *in, int ipv46_flag) { + char *constr = xmalloc(64); + switch (ipv46_flag) { + case IPV6_FLAG: { + char *err = (char *) inet_ntop(AF_INET6, in, constr, 64); + if (err == NULL) { + return NULL; + } + return constr; + } + case IPV4_FLAG: { + char *err = (char *) inet_ntop(AF_INET, in, constr, 64); + if (err == NULL) { + return NULL; + } + return constr; + } + default: + return NULL; + } +} + +int64_t get_file_lines(char *fileName) { + assert(strlen(fileName) < 1024); + char buff[1030] = {0x00}; + FILE *fp = NULL; + strcpy(buff, "wc -l "); + strcat(buff, fileName); + + fp = popen(buff, "r"); + if (fp) { + memset(buff, 0x00, sizeof(buff)); + if (fgets(buff, sizeof(buff), fp)) { + pclose(fp); + return strtoll(buff, NULL, 10); + } + } + if (fp) pclose(fp); + + return -1; +} + +#if defined(__APPLE__) + +#include + +#endif + +int drop_privs() { + struct passwd *pw; + + if (geteuid() != 0) { + /* Not root */ + return EXIT_SUCCESS; + } + + if ((pw = getpwnam("nobody")) != NULL) { + if (setuid(pw->pw_uid) == 0) { + return EXIT_SUCCESS; // success + } + } + + return EXIT_FAILURE; +} + +#if defined(__APPLE__) + +#include + +int set_cpu(uint32_t core) { + mach_port_t tid = pthread_mach_thread_np(pthread_self()); + struct thread_affinity_policy policy; + policy.affinity_tag = core; + + kern_return_t ret = thread_policy_set(tid, THREAD_AFFINITY_POLICY, + (thread_policy_t) &policy, + THREAD_AFFINITY_POLICY_COUNT); + if (ret != KERN_SUCCESS) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +#elif defined(__DragonFly__) +#include + +int set_cpu(uint32_t core) { + if (usched_set(getpid(), USCHED_SET_CPU, &core, sizeof(core)) != 0) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +#elif defined(__NetBSD__) +int set_cpu(uint32_t core) { + cpuset_t *cpuset = cpuset_create(); + if (cpuset == NULL) { + return EXIT_FAILURE; + } + + cpuset_zero(cpuset); + cpuset_set(core, cpuset); + cpuset_destroy(cpuset); + + if (pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), cpuset) != + 0) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +#else + +#if defined(__FreeBSD__) +#include +#include +#include +#define cpu_set_t cpuset_t +#endif + +int set_cpu(uint32_t core) { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(core, &cpuset); + + if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) != + 0) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +#endif diff --git a/lib/util.h b/lib/util.h new file mode 100644 index 0000000..fd493d9 --- /dev/null +++ b/lib/util.h @@ -0,0 +1,62 @@ +#ifndef XMAP_UTIL_H +#define XMAP_UTIL_H + +#include +#include + +#include "types.h" + +#define IPV6_FLAG 6 +#define IPV4_FLAG 4 + +int max_int(int a, int b); + +uint32_t parse_max_hosts(char *max_targets); + +void enforce_range(const char *name, int v, int min, int max); + +// Splits comma delimited string into char*[]. Does not handle +// escaping or complicated setups - designed to process a set +// of fields that the user wants output +void split_string(char *in, int *len, char ***results); + +// Print a string using w length long lines, attempting to break on +// spaces +void fprintw(FILE *f, char *s, size_t w); + +// pretty print elapsed (or estimated) number of seconds +void time_string(uint32_t time, int est, char *buf, size_t len); + +// pretty print quantities +void number_string(uint64_t n, char *buf, size_t len); + +// pretty print quantities (bandwidth) +void bits_string(uint64_t n, char *buf, size_t len); + +// Convert a string representation of a MAC address to a byte array +int parse_mac(macaddr_t *out, char *in); + +int check_range(int v, int min, int max); + +int file_exists(char *name); + +// If running as root, drops privileges to that of user "nobody". +// Otherwise, does nothing. +int drop_privs(); + +// Set CPU affinity to a single core +int set_cpu(uint32_t core); + +// IP string to IP address struct +int inet_str2in(const char *str, void *in, int ipv46_flag); + +// IP address struct to IP string +int inet_in2str(const void *in, char *str, int len, int ipv46_flag); + +// Note: caller must free return value +char *inet_in2constr(const void *in, int ipv46_flag); + +// get file lines +int64_t get_file_lines(char *fileName); + +#endif // XMAP_UTIL_H diff --git a/lib/xalloc.c b/lib/xalloc.c new file mode 100644 index 0000000..08739bc --- /dev/null +++ b/lib/xalloc.c @@ -0,0 +1,40 @@ +#include "xalloc.h" + +#include +#include + +#include "logger.h" + +static void die() __attribute__((noreturn)); + +void *xcalloc(size_t count, size_t size) { + void *res = calloc(count, size); + if (res == NULL) { + die(); + } + + return res; +} + +void xfree(void *ptr) { free(ptr); } + +void *xmalloc(size_t size) { + void *res = malloc(size); + if (res == NULL) { + die(); + } + memset(res, 0, size); + + return res; +} + +void *xrealloc(void *ptr, size_t size) { + void *res = realloc(ptr, size); + if (res == NULL) { + die(); + } + + return res; +} + +void die() { log_fatal("xmap", "Out of memory"); } diff --git a/lib/xalloc.h b/lib/xalloc.h new file mode 100644 index 0000000..3507db3 --- /dev/null +++ b/lib/xalloc.h @@ -0,0 +1,14 @@ +#ifndef XMAP_XALLOC_H +#define XMAP_XALLOC_H + +#include + +void *xcalloc(size_t count, size_t size); + +void xfree(void *ptr); + +void *xmalloc(size_t size); + +void *xrealloc(void *ptr, size_t size); + +#endif // XMAP_XALLOC_H diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..5cdb0de --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,6 @@ +These scripts are used by Travis CI to prepare the build environment. + +If you run them on your system, you should both + +1. Trust me, because they need sudo +2. Run them from the parent directory, e.g. `./scripts/install_cmake.sh` diff --git a/scripts/check_manfile.py b/scripts/check_manfile.py new file mode 100755 index 0000000..5e477ad --- /dev/null +++ b/scripts/check_manfile.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# +# CI runs this script to verify that options appearing in XTools' ggo.in files also appear in their .ronn files. +# It does not check that `make manpages` has actually been run. +# +# This script assumes it's being run from the root of the xmap repository. +# + +import sys + +checks = [ + ("xopt.ggo.in", "xmap.1.ronn") +] + +failures = False + +for ggo, ronn in checks: + options = [] + with open("src/" + ggo) as fd: + for l in fd: + if l.startswith("option "): + option = l.split()[1].lstrip('"').rstrip('"') + options.append(option) + + man = open("src/" + ronn).read() + + for option in options: + if option not in man: + failures = True + sys.stderr.write("option %s is present in %s but missing from man file %s\n" % (option, ggo, ronn)) + +if failures: + sys.exit(1) diff --git a/scripts/cmake_install_checksum b/scripts/cmake_install_checksum new file mode 100644 index 0000000..6fd3167 --- /dev/null +++ b/scripts/cmake_install_checksum @@ -0,0 +1 @@ +d0f8f9a6c921ba4927e69c68aa95a9a3 cmake_installer.sh diff --git a/scripts/install_cmake.py b/scripts/install_cmake.py new file mode 100644 index 0000000..fef8be0 --- /dev/null +++ b/scripts/install_cmake.py @@ -0,0 +1,50 @@ +import sys +import os +import os.path + +import sh +from sh import git, cd, make, rm, sudo, cp, chmod, mkdir + +def write_output(line): + sys.stdout.write(line) + +curl = sh.Command("curl") +tar = sh.Command("tar") + +install_env = os.environ.copy() +install_env['CC'] = "gcc" + +directory = os.path.dirname(os.path.realpath(__file__)) + +# Download it +cd(directory) +curl( + "-L", + "http://www.cmake.org/files/v3.2/cmake-3.2.2-Linux-x86_64.sh", + _out="cmake_installer.sh" +) + +# Set up the installer +installer_path = os.path.join(directory, "cmake_installer.sh") +chmod("a+x", installer_path) +cmake_installer = sh.Command(installer_path) + +# Verify the download +sum_str = sh.Command("openssl").sha1(installer_path) +expected_sum = "925e6185e94b717760453427b857fc4f2a4c2149" +if sum_str.split()[1] != expected_sum: + raise Exception + +# Install it +print("Installing...") +if os.environ.get("XMAP_TRAVIS_BUILD", None): + print("Travis CI build, installing to /opt") + with sudo: + cmake_installer(prefix="/opt", exclude_subdir=True) +else: + prefix = os.path.join(directory, "cmake") + mkdir(prefix) + print("Installing to {}".format(prefix)) + cmake_installer(prefix=prefix, exclude_subdir=True) + +print("Done.") diff --git a/scripts/install_jsonc.py b/scripts/install_jsonc.py new file mode 100644 index 0000000..14a5a82 --- /dev/null +++ b/scripts/install_jsonc.py @@ -0,0 +1,46 @@ +import sys +import os +import os.path + +import sh +from sh import git, cd, make, rm, sudo, cp + +def write_output(line): + sys.stdout.write(line) + +curl = sh.Command("curl") +tar = sh.Command("tar") + +install_env = os.environ.copy() +install_env['CC'] = "gcc" + +directory = os.path.dirname(os.path.realpath(__file__)) + +json_c_dir = os.path.join(directory, "json-c-json-c-0.12-20140410") +rm("-r", "-f", json_c_dir) + +cd(directory) +tar(curl( + "-L", + "https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz", + _piped=True +), "-xz") + +# Replace the Makefile.am.inc with one without -Werror +replacement_amfile = os.path.join(directory, "json_c_new_Makefile.am.inc") +original_amfile = os.path.join(json_c_dir, "Makefile.am.inc") +cp(replacement_amfile, original_amfile) + +# Build it +cd(json_c_dir) +autogen_location = os.path.join(json_c_dir, "autogen.sh") +autogen = sh.Command(autogen_location) +autogen(prefix="/usr", _out=write_output, _env=install_env) +make(_out=write_output, _env=install_env) + +if os.environ.get("XMAP_TRAVIS_BUILD", None): + print("Installing...") + with sudo: + make.install(_out=write_output, _env=install_env) + +print("Done.") diff --git a/scripts/install_mongo.py b/scripts/install_mongo.py new file mode 100644 index 0000000..62b8e43 --- /dev/null +++ b/scripts/install_mongo.py @@ -0,0 +1,38 @@ +import sys +import os +import os.path + +import sh +from sh import git, cd, make, rm, sudo + +def write_output(line): + sys.stdout.write(line) + +install_env = os.environ.copy() +install_env['CC'] = "gcc" + +directory = os.path.dirname(os.path.realpath(__file__)) + +mongo_c_driver = os.path.join(directory, "mongo-c-driver") + +rm("-r", "-f", mongo_c_driver) +autogen_location = os.path.join(mongo_c_driver, "autogen.sh") + +git.clone("https://github.com/mongodb/mongo-c-driver.git", + mongo_c_driver, + branch="1.17.4", + depth="1", + _out=write_output, + ) + +cd(mongo_c_driver) +autogen = sh.Command(autogen_location) +autogen(prefix="/usr", _out=write_output, _env=install_env) +make(_out=write_output, _env=install_env) + +if os.environ.get("XMAP_TRAVIS_BUILD", None): + print("Installing...") + with sudo: + make.install(_out=write_output, _env=install_env) + +print("Done.") diff --git a/scripts/json_c_new_Makefile.am.inc b/scripts/json_c_new_Makefile.am.inc new file mode 100644 index 0000000..c1fac9a --- /dev/null +++ b/scripts/json_c_new_Makefile.am.inc @@ -0,0 +1 @@ +AM_CFLAGS = -Wall -Wno-error=deprecated-declarations -Wextra -Wwrite-strings -Wno-unused-parameter -std=gnu99 -D_GNU_SOURCE -D_REENTRANT diff --git a/scripts/uninstall.cmake b/scripts/uninstall.cmake new file mode 100644 index 0000000..a303d93 --- /dev/null +++ b/scripts/uninstall.cmake @@ -0,0 +1,24 @@ +set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt") + +if(NOT EXISTS ${MANIFEST}) + message(FATAL_ERROR "Cannot find install manifest: '${MANIFEST}'") +endif() + +file(STRINGS ${MANIFEST} files) +foreach(file ${files}) + if(EXISTS ${file}) + message(STATUS "Removing file: '${file}'") + + exec_program( + ${CMAKE_COMMAND} ARGS "-E remove ${file}" + OUTPUT_VARIABLE stdout + RETURN_VALUE result + ) + + if(NOT "${result}" STREQUAL 0) + message(FATAL_ERROR "Failed to remove file: '${file}'.") + endif() + else() + MESSAGE(STATUS "File '${file}' does not exist.") + endif() +endforeach(file) \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..3f59e14 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,239 @@ +include_directories( + "${CMAKE_CURRENT_BINARY_DIR}" + ${PROJECT_SOURCE_DIR}/lib + ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/src/output_modules + ${PROJECT_SOURCE_DIR}/src/tests +) + +# ADD YOUR OUTPUT MODULE HERE +set(EXTRA_SOURCES_OUTPUT_MODULES + ) + +# ADD YOUR PROBE MODULE HERE +set(EXTRA_SOURCES_PROBE_MODULES + ) + +# ADD YOUR IID MODULE HERE +set(EXTRA_SOURCES_IID_MODULES + ) + +set(SOURCES_OUTPUT_MODULES + output_modules/output_modules.c + output_modules/module_csv.c + ) + +# Handle optional output modules +if(WITH_JSON) + set(SOURCES_OUTPUT_MODULES + ${SOURCES_OUTPUT_MODULES} + output_modules/module_json.c + ) +endif() + +if(WITH_REDIS) + set(SOURCES_OUTPUT_MODULES + ${SOURCES_OUTPUT_MODULES} + output_modules/module_redis_csv.c + output_modules/module_redis_packed.c + ) +endif() + +set(SOURCES_PROBE_MODULES + probe_modules/packet.c + probe_modules/packet_icmp.c + probe_modules/packet_icmp6.c + probe_modules/probe_modules.c + probe_modules/module_icmp_echo.c + probe_modules/module_icmp6_echo.c + probe_modules/module_icmp6_echo_gw.c + probe_modules/module_icmp6_echo_tmxd.c + probe_modules/module_tcp_syn.c + probe_modules/module_tcp6_syn.c + probe_modules/module_udp.c + probe_modules/module_udp6.c + ) + +set(SOURCES_IID_MODULES + iid_modules/iid_modules.c + iid_modules/module_full.c + iid_modules/module_low.c + iid_modules/module_low_fill.c + iid_modules/module_rand.c + iid_modules/module_set.c + iid_modules/module_zero.c + ) + +set(SOURCES + aesrand.c + cyclic.c + expression.c + fieldset.c + filter.c + get_gateway.c + iterator.c + ip_target_file.c + monitor.c + recv.c + send.c + shard.c + socket.c + state.c + summary.c + utility.c + validate.c + xmap.c + xopt_include.c + "${CMAKE_CURRENT_BINARY_DIR}/xopt.h" + "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" + "${CMAKE_CURRENT_BINARY_DIR}/parser.c" + ${SOURCES_IID_MODULES} + ${SOURCES_OUTPUT_MODULES} + ${SOURCES_PROBE_MODULES} + ${EXTRA_SOURCES_IID_MODULES} + ${EXTRA_SOURCES_OUTPUT_MODULES} + ${EXTRA_SOURCES_PROBE_MODULES} + ) + +#set(SOURCES_XTEST +# aesrand.c +# cyclic.c +# expression.c +# fieldset.c +# filter.c +# get_gateway.c +# iterator.c +# ip_target_file.c +# monitor.c +# recv.c +# send.c +# shard.c +# socket.c +# state.c +# summary.c +# utility.c +# validate.c +# tests/xtest.c +# xopt_include.c +# "${CMAKE_CURRENT_BINARY_DIR}/xopt.h" +# "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" +# "${CMAKE_CURRENT_BINARY_DIR}/parser.c" +# ${SOURCES_IID_MODULES} +# ${SOURCES_OUTPUT_MODULES} +# ${SOURCES_PROBE_MODULES} +# ${EXTRA_SOURCES_IID_MODULES} +# ${EXTRA_SOURCES_OUTPUT_MODULES} +# ${EXTRA_SOURCES_PROBE_MODULES} +# ) + +# Handle various versions of socket +if (WITH_PFRING) + set(SOURCES ${SOURCES} socket-pfring.c) + # set(SOURCES_XTEST ${SOURCES_XTEST} socket-pfring.c) +elseif (APPLE OR BSD) + set(SOURCES ${SOURCES} socket-bsd.c) + # set(SOURCES_XTEST ${SOURCES_XTEST} socket-bsd.c) +else () + set(SOURCES ${SOURCES} socket-linux.c) + # set(SOURCES_XTEST ${SOURCES_XTEST} socket-linux.c) +endif () + +# Handle various versions of recv +if (WITH_PFRING) + set(SOURCES ${SOURCES} recv-pfring.c) + # set(SOURCES_XTEST ${SOURCES_XTEST} recv-pfring.c) +else () + set(SOURCES ${SOURCES} recv-pcap.c) + # set(SOURCES_XTEST ${SOURCES_XTEST} recv-pcap.c) +endif () + +# Set configure time xmap version +configure_file(xopt.ggo.in ${CMAKE_BINARY_DIR}/src/xopt.ggo @ONLY) +# Additional ggo.in's should be added here and CMakeVersion.txt + +# This sets a *build* time dependency that checks git +if("${XMAP_VERSION}" STREQUAL "DEVELOPMENT") + add_custom_target(git_versioning ALL + COMMAND ${CMAKE_COMMAND} -D ORIG_SRC_DIR:STRING="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/src/CMakeVersion.txt" + ) +endif() + +add_custom_command( + OUTPUT xopt.h + COMMAND gengetopt -C --no-help --no-version --unamed-opts=SUBNETS -i "${CMAKE_CURRENT_BINARY_DIR}/xopt.ggo" -F "${CMAKE_CURRENT_BINARY_DIR}/xopt" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/xopt.ggo" +) + +add_custom_command( + OUTPUT lexer.c + COMMAND flex -o "${CMAKE_CURRENT_BINARY_DIR}/lexer.c" --header-file="${CMAKE_CURRENT_BINARY_DIR}/lexer.h" "${CMAKE_CURRENT_SOURCE_DIR}/lexer.l" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/lexer.l" +) + +if (NetBSD) + add_custom_command( + OUTPUT parser.c + COMMAND yacc -d -o "${CMAKE_CURRENT_BINARY_DIR}/parser.c" "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + ) +else () + add_custom_command( + OUTPUT parser.c + COMMAND byacc -d -o "${CMAKE_CURRENT_BINARY_DIR}/parser.c" "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/parser.y" + ) +endif () + +add_custom_target(manpages + ronn "${CMAKE_CURRENT_SOURCE_DIR}/xmap.1.ronn" --organization="XMap" --manual="xmap" + SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/xmap.1.ronn" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + ) + +# gen target +#add_executable(xtest +# ${SOURCES_XTEST} +# ) + +add_executable(xmap + ${SOURCES} + ) + +if (APPLE OR BSD) + set(DNET_LIBRARIES "dnet") +else () + set(DNET_LIBRARIES "") +endif () + +# lib for gen target +#target_link_libraries(xtest +# xmaplib +# ${PFRING_LIBRARIES} +# pcap gmp m unistring +# ${DNET_LIBRARIES} +# ${REDIS_LIBS} +# ${JSON_LIBRARIES} +# ) + +target_link_libraries(xmap + xmaplib + ${PFRING_LIBRARIES} + pcap gmp m unistring + ${DNET_LIBRARIES} + ${REDIS_LIBS} + ${JSON_LIBRARIES} + ) + +# Install binary +install( + TARGETS + xmap + RUNTIME DESTINATION sbin +) + +# Install Manpages +install( + FILES + xmap.1 + DESTINATION share/man/man1 +) diff --git a/src/CMakeVersion.txt b/src/CMakeVersion.txt new file mode 100644 index 0000000..b877f8e --- /dev/null +++ b/src/CMakeVersion.txt @@ -0,0 +1,11 @@ +set(GIT_CMD "git") +set(GIT_ARGS "log" "-n" "1" "--pretty=format:%h - %ad") +execute_process(COMMAND ${GIT_CMD} ${GIT_ARGS} + RESULT_VARIABLE GIT_RESULT + OUTPUT_VARIABLE GIT_COMMIT) + if (GIT_RESULT) + set (GIT_COMMIT "UNKNOWN") + endif() +set(XMAP_VERSION "Development Build. Commit ${GIT_COMMIT}") + +configure_file("${ORIG_SRC_DIR}/src/xopt.ggo.in" "${CMAKE_BINARY_DIR}/xopt.ggo" @ONLY) diff --git a/src/aesrand.c b/src/aesrand.c new file mode 100644 index 0000000..522928a --- /dev/null +++ b/src/aesrand.c @@ -0,0 +1,78 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "aesrand.h" + +#include + +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/rijndael-alg-fst.h" +#include "../lib/xalloc.h" + +#define AES_ROUNDS 10 +#define AES_BLOCK_WORDS 4 +#define AES_KEY_BYTES 16 +#define AES_KEY_BITS (AES_KEY_BYTES * 8) +#define OUTPUT_BYTES 16 + +struct aesrand { + uint32_t input[AES_BLOCK_WORDS]; + uint32_t sched[(AES_ROUNDS + 1) * 4]; + uint8_t output[OUTPUT_BYTES]; +}; + +static aesrand_t *_aesrand_init(uint8_t *key) { + aesrand_t *aes = xmalloc(sizeof(aesrand_t)); + memset(&aes->input, 0, AES_BLOCK_WORDS * 4); + if (rijndaelKeySetupEnc(aes->sched, key, AES_KEY_BITS) != AES_ROUNDS) { + log_fatal("aesrand", "could not initialize AES key"); + } + memset(aes->output, 0, OUTPUT_BYTES); + + return aes; +} + +aesrand_t *aesrand_init_from_seed(uint64_t seed) { + uint8_t key[AES_KEY_BYTES]; + memset(key, 0, AES_KEY_BYTES); + for (uint8_t i = 0; i < sizeof(seed); ++i) { + key[i] = (uint8_t)((seed >> 8 * i) & 0xFF); + } + + return _aesrand_init(key); +} + +aesrand_t *aesrand_init_from_random() { + uint8_t key[AES_KEY_BYTES]; + if (!random_bytes(key, AES_KEY_BYTES)) { + log_fatal("aesrand", "Couldn't get random bytes"); + } + + return _aesrand_init(key); +} + +uint64_t aesrand_getword(aesrand_t *aes) { + memcpy(aes->input, aes->output, sizeof(aes->input)); + rijndaelEncrypt(aes->sched, AES_ROUNDS, (uint8_t *) aes->input, + aes->output); + uint64_t retval; + memcpy(&retval, aes->output, sizeof(retval)); + + return retval; +} + +void aesrand_get128bits(mpz_t rop, aesrand_t *aes) { + memcpy(aes->input, aes->output, sizeof(aes->input)); + rijndaelEncrypt(aes->sched, AES_ROUNDS, (uint8_t *) aes->input, + aes->output); + mpz_import(rop, OUTPUT_BYTES, 1, 1, 0, 0, aes->output); +} diff --git a/src/aesrand.h b/src/aesrand.h new file mode 100644 index 0000000..4968fa1 --- /dev/null +++ b/src/aesrand.h @@ -0,0 +1,30 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_AESRAND_H +#define XMAP_AESRAND_H + +#include +#include + +typedef struct aesrand aesrand_t; + +aesrand_t *aesrand_init_from_random(); + +aesrand_t *aesrand_init_from_seed(uint64_t); + +uint64_t aesrand_getword(aesrand_t *aes); + +void aesrand_get128bits(mpz_t rop, aesrand_t *aes); + +aesrand_t *aesrand_free(aesrand_t *aes); + +#endif // XMAP_AESRAND_H diff --git a/src/cyclic.c b/src/cyclic.c new file mode 100644 index 0000000..84c8856 --- /dev/null +++ b/src/cyclic.c @@ -0,0 +1,636 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "cyclic.h" + +#include +#include + +#include "../lib/logger.h" + +#define GROUPS_SIZE 36 + +static cyclic_group_t groups[GROUPS_SIZE]; + +static _cyclic_group_t _groups[GROUPS_SIZE] = { + { + // 2^4 + 1 + .prime = "17", + .known_primroot = "3", + .prime_factors = + { + "2", + }, + .num_prime_factors = 1, + }, + { + // 2^8 + 1 + .prime = "257", + .known_primroot = "3", + .prime_factors = + { + "2", + }, + .num_prime_factors = 1, + }, + { + // 2^12 + 3 + .prime = "4099", + .known_primroot = "2", + .prime_factors = + { + "2", + "3", + "683", + }, + .num_prime_factors = 3, + }, + { + // 2^16 + 1 + .prime = "65537", + .known_primroot = "3", + .prime_factors = + { + "2", + }, + .num_prime_factors = 1, + }, + { + // 2^20 + 7 + .prime = "1048583", + .known_primroot = "5", + .prime_factors = + { + "101", + "179", + "2", + "29", + }, + .num_prime_factors = 4, + }, + { + // 2^24 + 43 + .prime = "16777259", + .known_primroot = "2", + .prime_factors = + { + "103", + "2", + "23", + "3541", + }, + .num_prime_factors = 4, + }, + { + // 2^28 + 3 + .prime = "268435459", + .known_primroot = "2", + .prime_factors = + { + "19", + "2", + "3", + "87211", + }, + .num_prime_factors = 4, + }, + { + // 2^32 + 15 + .prime = "4294967311", + .known_primroot = "3", + .prime_factors = + { + "131", + "2", + "3", + "364289", + "5", + }, + .num_prime_factors = 5, + }, + { + // 2^36 + 31 + .prime = "68719476767", + .known_primroot = "5", + .prime_factors = + { + "163", + "2", + "238727", + "883", + }, + .num_prime_factors = 4, + }, + { + // 2^40 + 15 + .prime = "1099511627791", + .known_primroot = "3", + .prime_factors = + { + "2", + "3", + "36650387593", + "5", + }, + .num_prime_factors = 4, + }, + { + // 2^44 + 7 + .prime = "17592186044423", + .known_primroot = "5", + .prime_factors = + { + "11", + "155542661", + "2", + "53", + "97", + }, + .num_prime_factors = 5, + }, + { + // 2^48 + 21 + .prime = "281474976710677", + .known_primroot = "6", + .prime_factors = + { + "1361", + "2", + "2462081249", + "3", + "7", + }, + .num_prime_factors = 5, + }, + { + // 2^52 + 21 + .prime = "4503599627370517", + .known_primroot = "2", + .prime_factors = + { + "2", + "23", + "3", + "612229", + "987127", + }, + .num_prime_factors = 5, + }, + { + // 2^56 + 81 + .prime = "72057594037928017", + .known_primroot = "10", + .prime_factors = + { + "14557303", + "2", + "3", + "34501", + "61", + "7", + }, + .num_prime_factors = 6, + }, + { + // 2^60 + 33 + .prime = "1152921504606847009", + .known_primroot = "13", + .prime_factors = + { + "11", + "2", + "2971", + "3", + "48912491", + "683", + }, + .num_prime_factors = 6, + }, + { + // 2^64 + 13 + .prime = "18446744073709551629", + .known_primroot = "2", + .prime_factors = + { + "2", + "658812288346769701", + "7", + }, + .num_prime_factors = 3, + }, + { + // 2^68 + 33 + .prime = "295147905179352825889", + .known_primroot = "29", + .prime_factors = + { + "19", + "2", + "3", + "43", + "5419", + "77158673929", + }, + .num_prime_factors = 6, + }, + { + // 2^72 + 15 + .prime = "4722366482869645213711", + .known_primroot = "6", + .prime_factors = + { + "2", + "3", + "4799", + "5", + "541316653", + "60594931", + }, + .num_prime_factors = 6, + }, + { + // 2^76 + 15 + .prime = "75557863725914323419151", + .known_primroot = "3", + .prime_factors = + { + "2", + "3", + "5", + "518763225032024191", + "971", + }, + .num_prime_factors = 5, + }, + { + // 2^80 + 13 + .prime = "1208925819614629174706189", + .known_primroot = "2", + .prime_factors = + { + "1093", + "2", + "31039", + "8908647580887961", + }, + .num_prime_factors = 4, + }, + { + // 2^84 + 3 + .prime = "19342813113834066795298819", + .known_primroot = "2", + .prime_factors = + { + "1163", + "13455809771", + "155377", + "2", + "2657", + "3", + "499", + }, + .num_prime_factors = 7, + }, + { + // 2^88 + 7 + .prime = "309485009821345068724781063", + .known_primroot = "5", + .prime_factors = + { + "2", + "228791153858291", + "676348286641", + }, + .num_prime_factors = 3, + }, + { + // 2^92 + 25 + .prime = "4951760157141521099596496921", + .known_primroot = "3", + .prime_factors = + { + "11411", + "2", + "296131229", + "5", + "5233517194231", + "7", + }, + .num_prime_factors = 6, + }, + { + // 2^96 + 61 + .prime = "79228162514264337593543950397", + .known_primroot = "2", + .prime_factors = + { + "1032208068610458304152691", + "2", + "31", + "619", + }, + .num_prime_factors = 4, + }, + { + // 2^100 + 277 + .prime = "1267650600228229401496703205653", + .known_primroot = "2", + .prime_factors = + { + "2", + "52203989", + "6070659658921032842417", + }, + .num_prime_factors = 3, + }, + { + // 2^104 + 111 + .prime = "20282409603651670423947251286127", + .known_primroot = "3", + .prime_factors = + { + "2", + "3", + "3544157", + "953795670058807140125153", + }, + .num_prime_factors = 4, + }, + { + // 2^108 + 33 + .prime = "324518553658426726783156020576289", + .known_primroot = "13", + .prime_factors = + { + "2", + "3", + "415141630193", + "8142767081771726171", + }, + .num_prime_factors = 4, + }, + { + // 2^112 + 25 + .prime = "5192296858534827628530496329220121", + .known_primroot = "3", + .prime_factors = + { + "1201", + "2", + "28976429", + "3730024228806121761107", + "5", + }, + .num_prime_factors = 5, + }, + { + // 2^116 + 33 + .prime = "83076749736557242056487941267521569", + .known_primroot = "23", + .prime_factors = + { + "107775231312019", + "17539", + "1777", + "2", + "25781083", + "3", + "3331", + }, + .num_prime_factors = 7, + }, + { + // 2^120 + 451 + .prime = "1329227995784915872903807060280345027", + .known_primroot = "2", + .prime_factors = + { + "1097", + "11", + "1361", + "170777", + "2", + "233", + "322747", + "41", + "76856512570457", + }, + .num_prime_factors = 9, + }, + { + // 2^124 + 67 + .prime = "21267647932558653966460912964485513283", + .known_primroot = "2", + .prime_factors = + { + "1216997", + "187385987", + "2", + "30265284680791", + "37578049", + "41", + }, + .num_prime_factors = 6, + }, + { + // 2^128 + 51 + .prime = "340282366920938463463374607431768211507", + .known_primroot = "2", + .prime_factors = + { + "12275703273579557140363", + "17", + "2", + "3", + "5816689", + "6481", + "89", + }, + .num_prime_factors = 7, + }, + { + // 2^132 + 67 + .prime = "5444517870735015415413993718908291383363", + .known_primroot = "2", + .prime_factors = + { + "1203053633927087259807466469264791", + "2", + "33773", + "67", + }, + .num_prime_factors = 4, + }, + { + // 2^136 + 85 + .prime = "87112285931760246646623899502532662132821", + .known_primroot = "2", + .prime_factors = + { + "17", + "2", + "2971", + "5", + "909288538628219561960108944043", + "94841", + }, + .num_prime_factors = 6, + }, + { + // 2^140 + 37 + .prime = "1393796574908163946345982392040522594123813", + .known_primroot = "2", + .prime_factors = + { + "2", + "577", + "603897996060729612801552162929169234889", + }, + .num_prime_factors = 3, + }, + { + // 2^144 + 175 + .prime = "22300745198530623141535718272648361505980591", + .known_primroot = "11", + .prime_factors = + { + "2", + "318582074264723187736224546752119450085437", + "5", + "7", + }, + .num_prime_factors = 4, + }, +}; + +#define MPZ_STR_BASE 10 + +// Generate groups from _groups +static void make_groups() { + for (size_t i = 0; i < GROUPS_SIZE; i++) { + mpz_init_set_str(groups[i].prime, _groups[i].prime, MPZ_STR_BASE); + mpz_init_set_str(groups[i].known_primroot, _groups[i].known_primroot, + MPZ_STR_BASE); + groups[i].num_prime_factors = _groups[i].num_prime_factors; + + for (size_t j = 0; j < _groups[i].num_prime_factors; j++) { + mpz_init_set_str(groups[i].prime_factors[j], + _groups[i].prime_factors[j], MPZ_STR_BASE); + } + } +} + +// Check whether an integer is coprime with (p - 1) +static int is_coprime(const mpz_t check, const cyclic_group_t *group) { + int flag = 0; + if (mpz_eq_ui(check, 0) || mpz_eq_ui(check, 1)) { + return flag; + } + + mpz_t t; + mpz_init(t); + for (size_t i = 0; i < group->num_prime_factors; i++) { + if (mpz_gt(group->prime_factors[i], check)) { + mpz_mod(t, group->prime_factors[i], check); + if (mpz_eq_ui(t, 0)) goto cleanup; + } else if (mpz_lt(group->prime_factors[i], check)) { + mpz_mod(t, check, group->prime_factors[i]); + if (mpz_eq_ui(t, 0)) goto cleanup; + } else + goto cleanup; + } + + flag = 1; +cleanup: + mpz_clear(t); + + return flag; +} + +// Perform the isomorphism from (Z/pZ)+ to (Z/pZ)* +// Given known primitive root of (Z/pZ)* n, with x in (Z/pZ)+, do: +// f(x) = n^x mod p +// +// The isomorphism in the reverse direction is discrete log, and is +// therefore hard. +static void isomorphism(mpz_t primroot, const mpz_t additive_elt, + const cyclic_group_t *mult_group) { + assert(mpz_lt(additive_elt, mult_group->prime)); + mpz_powm(primroot, mult_group->known_primroot, additive_elt, + mult_group->prime); + + log_debug("cyclic", "isomorphism: %s", mpz_to_str10(primroot)); +} + +// Return a (random) number coprime with (p - 1) of the group, +// which is a generator of the additive group mod (p - 1) +static void find_primroot(mpz_t primroot, const cyclic_group_t *group, + aesrand_t *aes) { + mpz_t candidate; + // mpz_init_set_ui(candidate, aesrand_getword(aes)); + mpz_init(candidate); + aesrand_get128bits(candidate, aes); + mpz_mod(candidate, candidate, group->prime); + + while (!is_coprime(candidate, group)) { + mpz_add_ui(candidate, candidate, 1); + mpz_mod(candidate, candidate, group->prime); + } + isomorphism(primroot, candidate, group); + log_debug("cyclic", "primitive root: %s", mpz_to_str10(primroot)); + mpz_clear(candidate); +} + +// Get a cyclic_group_t of at least min_size. +// Pointer into static data, do not free(). +const cyclic_group_t *get_group(const mpz_t min_size) { + make_groups(); + for (size_t i = 0; i < GROUPS_SIZE; i++) { + if (mpz_gt(groups[i].prime, min_size)) { + log_debug("cyclic", "prime: %s", mpz_to_str10(groups[i].prime)); + return &groups[i]; + } + } + + // Should not reach, final group should always be larger than 2^32 + assert(0); +} + +cycle_t make_cycle(const cyclic_group_t *group, aesrand_t *aes) { + cycle_t cycle; + + cycle.group = group; + mpz_init(cycle.generator); + find_primroot(cycle.generator, group, aes); + + mpz_t t; + // mpz_init_set_ui(t, aesrand_getword(aes)); + mpz_init(t); + aesrand_get128bits(t, aes); + mpz_init(cycle.offset); + mpz_mod(cycle.offset, t, group->prime); + log_debug("cyclic", "cycle offset: %s", mpz_to_str10(cycle.offset)); + mpz_clear(t); + + mpz_init(cycle.order); + mpz_sub_ui(cycle.order, group->prime, 1); + log_debug("cyclic", "cycle order: %s", mpz_to_str10(cycle.order)); + + return cycle; +} + +void close_cycle(cycle_t cycle) { + mpz_clear(cycle.generator); + mpz_clear(cycle.offset); + mpz_clear(cycle.order); + + log_debug("cycle", "cleaning up"); +} diff --git a/src/cyclic.h b/src/cyclic.h new file mode 100644 index 0000000..11aa710 --- /dev/null +++ b/src/cyclic.h @@ -0,0 +1,57 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_CYCLIC_H +#define XMAP_CYCLIC_H + +#include +#include +#include + +#include "aesrand.h" + +#define MAX_MPZ_STR_LEN 50 +#define MAX_PRIM_FACTOR_SIZE 10 + +// Represents a multiplicative cyclic group (Z/pZ)* +typedef struct _cyclic_group { + char prime[MAX_MPZ_STR_LEN]; // p + char known_primroot[MAX_MPZ_STR_LEN]; // Known primitive root of (Z/pZ)* + char prime_factors[MAX_PRIM_FACTOR_SIZE] + [MAX_MPZ_STR_LEN]; // Unique prime factors of (p-1) + size_t num_prime_factors; // size of num_prime_factors +} _cyclic_group_t; + +typedef struct cyclic_group { + mpz_t prime; // p + mpz_t known_primroot; // Known primitive root of (Z/pZ)* + mpz_t prime_factors[MAX_PRIM_FACTOR_SIZE]; // Unique prime factors of (p-1) + size_t num_prime_factors; // size of num_prime_factors +} cyclic_group_t; + +// Represents a cycle in a group +typedef struct cycle { + const cyclic_group_t *group; + mpz_t generator; + mpz_t order; + mpz_t offset; +} cycle_t; + +// Get a cyclic_group_t of at least min_size. +// Pointer into static data, do not free(). +const cyclic_group_t *get_group(const mpz_t min_size); + +// Generate cycle (find generator and inverse) +cycle_t make_cycle(const cyclic_group_t *group, aesrand_t *aes); + +void close_cycle(cycle_t cycle); + +#endif // XMAP_CYCLIC_H diff --git a/src/expression.c b/src/expression.c new file mode 100644 index 0000000..903f80b --- /dev/null +++ b/src/expression.c @@ -0,0 +1,195 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "expression.h" + +#include +#include + +#include "../lib/xalloc.h" + +/* Static helper functions */ + +static node_t *alloc_node(); + +static int eval_gt_node(node_t *node, fieldset_t *fields); + +static int eval_lt_node(node_t *node, fieldset_t *fields); + +static int eval_eq_node(node_t *node, fieldset_t *fields); + +static int eval_lt_eq_node(node_t *node, fieldset_t *fields); + +static int eval_gt_eq_node(node_t *node, fieldset_t *fields); + +static node_t *alloc_node() { + node_t *node = xmalloc(sizeof(node_t)); + + return node; +} + +static int eval_gt_node(node_t *node, fieldset_t *fields) { + int index = node->left_child->value.field.index; + uint64_t expected = node->right_child->value.int_literal; + uint64_t actual = fs_get_uint64_by_index(fields, index); + + return (actual > expected); +} + +static int eval_lt_node(node_t *node, fieldset_t *fields) { + int index = node->left_child->value.field.index; + uint64_t expected = node->right_child->value.int_literal; + uint64_t actual = fs_get_uint64_by_index(fields, index); + + return (actual < expected); +} + +static int eval_eq_node(node_t *node, fieldset_t *fields) { + node_t *literal = node->right_child; + int index = node->left_child->value.field.index; + char * expected, *actual; + switch (literal->type) { + case STRING: + expected = literal->value.string_literal; + actual = fs_get_string_by_index(fields, index); + return (strcmp(expected, actual) == 0); + break; + case INT: + return (fs_get_uint64_by_index(fields, index) == + literal->value.int_literal); + break; + default: + printf("wat\n"); + break; + } + + return 0; +} + +static int eval_lt_eq_node(node_t *node, fieldset_t *fields) { + return !(eval_gt_node(node, fields)); +} + +static int eval_gt_eq_node(node_t *node, fieldset_t *fields) { + return !(eval_lt_node(node, fields)); +} + +/* Exposed functions */ + +node_t *make_op_node(enum operation op) { + node_t *node = alloc_node(); + node->type = OP; + node->value.op = op; + + return node; +} + +node_t *make_field_node(char *fieldname) { + node_t *node = alloc_node(); + node->type = FIELD; + node->value.field.fieldname = fieldname; + + return node; +} + +node_t *make_string_node(char *literal) { + node_t *node = alloc_node(); + node->type = STRING; + node->value.string_literal = literal; + + return node; +} + +node_t *make_int_node(int literal) { + node_t *node = alloc_node(); + node->type = INT; + node->value.int_literal = literal; + + return node; +} + +int evaluate_expression(node_t *root, fieldset_t *fields) { + if (!root) return 1; + switch (root->type) { /* XXX Not sure if runs */ + case FIELD: + case STRING: + case INT: + return 1; + case OP: + break; + } + switch (root->value.op) { + case GT: + return eval_gt_node(root, fields); + case LT: + return eval_lt_node(root, fields); + case EQ: + return eval_eq_node(root, fields); + case NEQ: + return (!eval_eq_node(root, fields)); + case LT_EQ: + return eval_lt_eq_node(root, fields); + case GT_EQ: + return eval_gt_eq_node(root, fields); + case AND: + return (evaluate_expression(root->left_child, fields) && + evaluate_expression(root->right_child, fields)); + case OR: + return (evaluate_expression(root->left_child, fields) || + evaluate_expression(root->right_child, fields)); + } + + return 0; +} + +static const char *get_op_str(int op) { + switch (op) { + case GT: + return ">"; + case LT: + return "<"; + case EQ: + return "=="; + case NEQ: + return "!="; + case LT_EQ: + return "<="; + case GT_EQ: + return ">="; + case AND: + return "&&"; + case OR: + return "||"; + default: + return ""; + } +} + +void print_expression(node_t *root) { + if (!root) return; + printf("%s", "("); + print_expression(root->left_child); + switch (root->type) { + case OP: + printf(" %s ", get_op_str(root->value.op)); + break; + case FIELD: + printf("(%s", root->value.field.fieldname); + break; + case STRING: + printf("%s)", root->value.string_literal); + break; + case INT: + printf("%llu)", (long long unsigned) root->value.int_literal); + break; + default: + break; + } + print_expression(root->right_child); + printf("%s", ")"); +} diff --git a/src/expression.h b/src/expression.h new file mode 100644 index 0000000..05e51db --- /dev/null +++ b/src/expression.h @@ -0,0 +1,49 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_EXPRESSION_H +#define XMAP_EXPRESSION_H + +#include "fieldset.h" + +enum operation { GT, LT, EQ, NEQ, AND, OR, LT_EQ, GT_EQ }; + +enum node_type { OP, FIELD, STRING, INT }; + +struct field_id { + int index; + char *fieldname; +}; + +union node_value { + struct field_id field; + char * string_literal; + uint64_t int_literal; + enum operation op; +}; + +typedef struct node_st { + struct node_st * left_child; + struct node_st * right_child; + enum node_type type; + union node_value value; +} node_t; + +node_t *make_op_node(enum operation op); + +node_t *make_field_node(char *fieldname); + +node_t *make_string_node(char *literal); + +node_t *make_int_node(int literal); + +int evaluate_expression(node_t *root, fieldset_t *fields); + +void print_expression(node_t *root); + +#endif // XMAP_EXPRESSION_H diff --git a/src/fieldset.c b/src/fieldset.c new file mode 100644 index 0000000..cc644d7 --- /dev/null +++ b/src/fieldset.c @@ -0,0 +1,354 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "fieldset.h" + +#include +#include +#include + +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len) { + if (fds->len + len > MAX_FIELDS) { + log_fatal("fieldset", "out of room in field def set"); + } + + fielddef_t *open = &(fds->fielddefs[fds->len]); + memcpy(open, fs, len * sizeof(fielddef_t)); + fds->len += len; +} + +fieldset_t *fs_new_fieldset(void) { + fieldset_t *f = xcalloc(1, sizeof(fieldset_t)); + f->len = 0; + f->type = FS_FIELDSET; + + return f; +} + +fieldset_t *fs_new_repeated_field(int type, int free_) { + fieldset_t *f = xcalloc(1, sizeof(fieldset_t)); + f->len = 0; + f->type = FS_REPEATED; + f->inner_type = type; + f->free_ = free_; + + return f; +} + +fieldset_t *fs_new_repeated_uint64(void) { + return fs_new_repeated_field(FS_UINT64, 0); +} + +fieldset_t *fs_new_repeated_bool(void) { + return fs_new_repeated_field(FS_BOOL, 0); +} + +fieldset_t *fs_new_repeated_string(int free_) { + return fs_new_repeated_field(FS_STRING, free_); +} + +fieldset_t *fs_new_repeated_binary(int free_) { + return fs_new_repeated_field(FS_BINARY, free_); +} + +fieldset_t *fs_new_repeated_fieldset(void) { + return fs_new_repeated_field(FS_FIELDSET, 0); +} + +static inline void fs_add_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, field_val_t value) { + if (fs->len + 1 >= MAX_FIELDS) { + log_fatal("fieldset", "out of room in fieldset"); + } + if (fs->type == FS_REPEATED && fs->inner_type != type) { + log_fatal("fieldset", + "object added to repeated field does not match type " + "of repeated field."); + } + field_t *f = &(fs->fields[fs->len]); + fs->len++; + f->type = type; + f->name = name; + f->len = len; + f->value = value; + f->free_ = free_; +} + +static void fs_modify_word(fieldset_t *fs, const char *name, int type, + int free_, size_t len, field_val_t value) { + for (int i = 0; i < fs->len; i++) { + if (!strcmp(fs->fields[i].name, name)) { + if (fs->fields[i].free_) { + free(fs->fields[i].value.ptr); + fs->fields[i].value.ptr = NULL; + } + fs->fields[i].type = type; + fs->fields[i].free_ = free_; + fs->fields[i].len = len; + fs->fields[i].value = value; + return; + } + } + fs_add_word(fs, name, type, free_, len, value); +} + +static char *sanitize_utf8(const char *buf) { + const char *ptr = buf; + + // Count how many errors we encounter + uint32_t i = 0; + // Upper bounds to ensure termination even if u8_check is unsafe + while (i < strlen(buf) && ptr < buf + strlen(buf)) { + ptr = (char *) u8_check((uint8_t *) ptr, strlen(ptr)); + if (ptr == NULL) { + break; + } + + assert(ptr >= buf); + assert(ptr < buf + strlen(buf)); + + ptr++; + i++; + } + + // i is the total number of errors. We need 2 extra bytes for each rune + char *safe_buf = xmalloc(strlen(buf) + i * 2 + 1); + char *safe_ptr = NULL; + memcpy(safe_buf, buf, strlen(buf)); + + // Fix exactly i errors + for (uint32_t j = 0; j < i; j++) { + // Always operate on the working buffer + safe_ptr = (char *) u8_check((uint8_t *) safe_buf, strlen(safe_buf)); + + // This implies we had less errors than we should. + // This is temporary debug code. + if (safe_ptr == NULL) { + log_warn("fieldset", + "UTF8 Sanitization issue. %u errors, fell through iter " + "%u. Orig: %s new: %s", + i, j, buf, safe_buf); + i = j; + break; + } + + // XXX Uncomment when we remove above log_warn. + // assert(safe_ptr != NULL); + assert(safe_ptr >= safe_buf); + assert(safe_ptr < safe_buf + strlen(safe_buf)); + + // Shift the rest of the string by 2 bytes + if (strlen(safe_ptr) > 1) { + memcpy(safe_ptr + 3, safe_ptr + 1, strlen(safe_ptr + 1)); + } + + // UTF8 replacement rune + safe_ptr[0] = (char) 0xef; + safe_ptr[1] = (char) 0xbf; + safe_ptr[2] = (char) 0xbd; + } + + // We now have a valid utf8 string + assert(u8_check((uint8_t *) safe_buf, strlen(safe_buf)) == NULL); + // We should be null terminated + assert(safe_buf[strlen(buf) + i * 2] == '\0'); + // We should be the right length + assert(strlen(safe_buf) == (strlen(buf) + i * 2)); + + return safe_buf; +} + +void fs_add_null(fieldset_t *fs, const char *name) { + field_val_t val = {.ptr = NULL}; + fs_add_word(fs, name, FS_NULL, 0, 0, val); +} + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_) { + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_STRING, free_, strlen(value), val); +} + +void fs_add_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_) { + if (u8_check((uint8_t *) value, strlen(value)) == NULL) { + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_STRING, free_, strlen(value), val); + } else { + char *safe_value = sanitize_utf8(value); + + if (free_) { + free(value); + } + + field_val_t val = {.ptr = safe_value}; + fs_add_word(fs, name, FS_STRING, 1, strlen(safe_value), val); + } +} + +void fs_chkadd_string(fieldset_t *fs, const char *name, char *value, + int free_) { + if (value) { + fs_add_string(fs, name, value, free_); + } else { + fs_add_null(fs, name); + } +} + +void fs_chkadd_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_) { + if (value) { + fs_add_unsafe_string(fs, name, value, free_); + } else { + fs_add_null(fs, name); + } +} + +void fs_add_constchar(fieldset_t *fs, const char *name, const char *value) { + field_val_t val = {.ptr = (char *) value}; + fs_add_word(fs, name, FS_STRING, 0, strlen(value), val); +} + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value) { + field_val_t val = {.num = value}; + fs_add_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val); +} + +void fs_add_bool(fieldset_t *fs, const char *name, int value) { + field_val_t val = {.num = value}; + fs_add_word(fs, name, FS_BOOL, 0, sizeof(int), val); +} + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_) { + field_val_t val = {.ptr = value}; + fs_add_word(fs, name, FS_BINARY, free_, len, val); +} + +void fs_add_fieldset(fieldset_t *fs, const char *name, fieldset_t *child) { + field_val_t val = {.ptr = child}; + fs_add_word(fs, name, FS_FIELDSET, 1, sizeof(void *), val); +} + +void fs_add_repeated(fieldset_t *fs, const char *name, fieldset_t *child) { + field_val_t val = {.ptr = child}; + fs_add_word(fs, name, FS_REPEATED, 1, sizeof(void *), val); +} + +// Modify +void fs_modify_null(fieldset_t *fs, const char *name) { + field_val_t val = {.ptr = NULL}; + fs_modify_word(fs, name, FS_NULL, 0, 0, val); +} + +void fs_modify_string(fieldset_t *fs, const char *name, char *value, + int free_) { + field_val_t val = {.ptr = value}; + fs_modify_word(fs, name, FS_STRING, free_, strlen(value), val); +} + +void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value) { + field_val_t val = {.num = value}; + fs_modify_word(fs, name, FS_UINT64, 0, sizeof(uint64_t), val); +} + +void fs_modify_bool(fieldset_t *fs, const char *name, int value) { + field_val_t val = {.num = value}; + fs_modify_word(fs, name, FS_BOOL, 0, sizeof(int), val); +} + +void fs_modify_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_) { + field_val_t val = {.ptr = value}; + fs_modify_word(fs, name, FS_BINARY, free_, len, val); +} + +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index) { + return (uint64_t) fs->fields[index].value.num; +} + +char *fs_get_string_by_index(fieldset_t *fs, int index) { + return (char *) fs->fields[index].value.ptr; +} + +int fds_get_index_by_name(fielddefset_t *fds, char *name) { + for (int i = 0; i < fds->len; i++) { + if (!strcmp(fds->fielddefs[i].name, name)) { + return i; + } + } + + return -1; +} + +void field_free(field_t *f) { + if (f->type == FS_FIELDSET || f->type == FS_REPEATED) { + fs_free((fieldset_t *) f->value.ptr); + } else if (f->free_) { + free(f->value.ptr); + } +} + +void fs_free(fieldset_t *fs) { + if (!fs) { + return; + } + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + field_free(f); + } + free(fs); +} + +void fs_generate_fieldset_translation(translation_t *t, fielddefset_t *avail, + char **req, int reqlen) { + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", "unable to allocate memory for translation"); + } + for (int i = 0; i < reqlen; i++) { + int l = fds_get_index_by_name(avail, req[i]); + if (l < 0) { + log_fatal("fieldset", + "specified field (%s) not " + "available in selected " + "probe module.", + req[i]); + } + t->translation[t->len++] = l; + } +} + +void fs_generate_full_fieldset_translation(translation_t *t, + fielddefset_t *avail) { + memset(t, 0, sizeof(translation_t)); + if (!t) { + log_fatal("fieldset", "unable to allocate memory for translation"); + } + t->len = avail->len; + for (int i = 0; i < avail->len; i++) { + t->translation[i] = i; + } +} + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t) { + fieldset_t *retv = fs_new_fieldset(); + if (!retv) { + log_fatal("fieldset", + "unable to allocate space for translated field set"); + } + for (int i = 0; i < t->len; i++) { + int o = t->translation[i]; + memcpy(&(retv->fields[i]), &(fs->fields[o]), sizeof(field_t)); + } + retv->len = t->len; + + return retv; +} diff --git a/src/fieldset.h b/src/fieldset.h new file mode 100644 index 0000000..21974d8 --- /dev/null +++ b/src/fieldset.h @@ -0,0 +1,151 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_FIELDSET_H +#define XMAP_FIELDSET_H + +#include +#include + +// maximum number of records that can be stored in a fieldset +#define MAX_FIELDS 128 +#define MAX_LIST_LENGTH 255 + +// types of data that can be stored in a field +#define FS_RESERVED 0 +#define FS_STRING 1 +#define FS_UINT64 2 +#define FS_BINARY 3 +#define FS_NULL 4 +#define FS_BOOL 7 +// recursive support +#define FS_FIELDSET 5 +#define FS_REPEATED 6 + +// definition of a field that's provided by a probe module +// these are used so that users can ask at the command-line +// what fields are available for consumption +typedef struct field_def { + const char *name; + const char *type; + const char *desc; +} fielddef_t; + +typedef struct fielddef_set { + fielddef_t fielddefs[MAX_FIELDS]; + int len; +} fielddefset_t; + +typedef union field_val { + void * ptr; + uint64_t num; +} field_val_t; + +// the internal field type used by fieldset +typedef struct field { + const char *name; + int type; + int free_; + size_t len; + field_val_t value; +} field_t; + +// data structure that is populated by the probe module +// and translated into the data structure that's passed +// to the output module +typedef struct fieldset { + int len; + field_t fields[MAX_FIELDS]; + // only used for repeated. + int inner_type; // type of repeated element. e.g., FS_STRING + int type; // REPEATED or FIELDSET + int free_; // should elements be freed +} fieldset_t; + +// we pass a different fieldset to an output module than +// the probe module generates for us because a user may +// only want certain fields and will expect them in a certain +// order. We generate a translated fieldset that contains +// only the fields we want to export to the output module. +// a translation specifies how to efficiently convert the fs +// povided by the probe module to the fs for the output module. +typedef struct translation { + int len; + int translation[MAX_FIELDS]; +} translation_t; + +fieldset_t *fs_new_fieldset(void); + +fieldset_t *fs_new_repeated_field(int type, int free_); + +fieldset_t *fs_new_repeated_uint64(void); + +fieldset_t *fs_new_repeated_bool(void); + +fieldset_t *fs_new_repeated_string(int free_); + +fieldset_t *fs_new_repeated_binary(int free_); + +fieldset_t *fs_new_repeated_fieldset(); + +char *fs_get_string_by_index(fieldset_t *fs, int index); + +int fds_get_index_by_name(fielddefset_t *fds, char *name); + +void gen_fielddef_set(fielddefset_t *fds, fielddef_t fs[], int len); + +void fs_add_null(fieldset_t *fs, const char *name); + +void fs_add_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_add_bool(fieldset_t *fs, const char *name, int value); + +void fs_add_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_add_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_); + +void fs_chkadd_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_chkadd_unsafe_string(fieldset_t *fs, const char *name, char *value, + int free_); + +void fs_add_constchar(fieldset_t *fs, const char *name, const char *value); + +void fs_add_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_); + +void fs_add_fieldset(fieldset_t *fs, const char *name, fieldset_t *child); + +void fs_add_repeated(fieldset_t *fs, const char *name, fieldset_t *child); + +// Modify +void fs_modify_null(fieldset_t *fs, const char *name); + +void fs_modify_uint64(fieldset_t *fs, const char *name, uint64_t value); + +void fs_modify_bool(fieldset_t *fs, const char *name, int value); + +void fs_modify_string(fieldset_t *fs, const char *name, char *value, int free_); + +void fs_modify_binary(fieldset_t *fs, const char *name, size_t len, void *value, + int free_); + +uint64_t fs_get_uint64_by_index(fieldset_t *fs, int index); + +void fs_free(fieldset_t *fs); + +void fs_generate_fieldset_translation(translation_t *t, fielddefset_t *avail, + char **req, int reqlen); + +fieldset_t *translate_fieldset(fieldset_t *fs, translation_t *t); + +void fs_generate_full_fieldset_translation(translation_t *t, + fielddefset_t *avail); + +#endif // XMAP_FIELDSET_H diff --git a/src/filter.c b/src/filter.c new file mode 100644 index 0000000..5c31f30 --- /dev/null +++ b/src/filter.c @@ -0,0 +1,109 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "filter.h" + +#include + +#include "lexer.h" +#include "parser.h" +#include "state.h" + +#include "../lib/logger.h" + +extern int yyparse(); + +node_t *zfilter; + +static int validate_node(node_t *node, fielddefset_t *fields) { + int index, found = 0; + if (node->type == OP) { + // These end up getting validated later + if (node->value.op == AND || node->value.op == OR) { + return 1; + } + // Comparison node (=, >, <, etc.) + // Validate that the field (left child) exists in the fieldset + for (index = 0; index < fields->len; index++) { + if (fields->fielddefs[index].name) { + if (strcmp(fields->fielddefs[index].name, + node->left_child->value.field.fieldname) == 0) { + node->left_child->value.field.index = index; + found = 1; + break; + } + } + } + if (!found) { + fprintf(stderr, "Field '%s' does not exist\n", + node->left_child->value.field.fieldname); + return 0; + } + // Fieldname is fine, match the type. + switch (node->right_child->type) { + case STRING: + if (strcmp(fields->fielddefs[index].type, "string") == 0) { + return 1; + } else { + fprintf(stderr, "Field '%s' is not of type 'string'\n", + fields->fielddefs[index].name); + return 0; + } + case INT: + if (strcmp(fields->fielddefs[index].type, "int") == 0 || + strcmp(fields->fielddefs[index].type, "bool") == 0) { + return 1; + } else { + fprintf(stderr, "Field '%s' is not of type 'int'\n", + fields->fielddefs[index].name); + return 0; + } + default: + return 0; + } + } else { + // All non-op nodes are valid + return 1; + } + + // Didn't validate + return 0; +} + +int parse_filter_string(char *filter) { + YY_BUFFER_STATE buffer_state = yy_scan_string(filter); + int status = yyparse(); + yy_delete_buffer(buffer_state); + if (status) { + // Error + log_error("xmap", "Unable to parse filter string: '%s'", filter); + return 0; + } + xconf.filter.expression = zfilter; + + return 1; +} + +/* + * 0 Valid + * -1 Invalid Field Name + * -2 Type Mismatch + */ +int validate_filter(node_t *root, fielddefset_t *fields) { + int valid; + if (!root) { + return 1; + } + valid = validate_node(root, fields); + if (!valid) { + return 0; + } + + return (validate_filter(root->left_child, fields) && + validate_filter(root->right_child, fields)); +} diff --git a/src/filter.h b/src/filter.h new file mode 100644 index 0000000..ad9c00b --- /dev/null +++ b/src/filter.h @@ -0,0 +1,23 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_FILTER_H +#define XMAP_FILTER_H + +#include "expression.h" +#include "fieldset.h" + +struct output_filter { + node_t *expression; +}; + +int parse_filter_string(char *filter); + +int validate_filter(node_t *root, fielddefset_t *fields); + +#endif // XMAP_FILTER_H diff --git a/src/get_gateway-bsd.h b/src/get_gateway-bsd.h new file mode 100644 index 0000000..ebf3528 --- /dev/null +++ b/src/get_gateway-bsd.h @@ -0,0 +1,237 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_GET_GATEWAY_BSD_H +#define XMAP_GET_GATEWAY_BSD_H + +#ifdef XMAP_GET_GATEWAY_LINUX_H +#error "Don't include both get_gateway-bsd.h and get_gateway-linux.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC_MINOR_ >= 6 +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif +#endif + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#define ROUNDUP(a) \ + ((a) > 0 ? (1 + (((a) -1) | (sizeof(int) - 1))) : sizeof(int)) +#define UNUSED __attribute__((unused)) +#define ZCPREFIX "zc:" +#define ZCPREFIX_LEN 3 + +// TODO: not support to get default gw's ipv6 address +static int _get_default_gw(uint8_t *gw_ip, char **iface) { + char buf[4096]; + struct rt_msghdr *rtm = (struct rt_msghdr *) &buf; + memset(rtm, 0, sizeof(buf)); + int seq = 0x00FF; + rtm->rtm_msglen = sizeof(buf); + rtm->rtm_type = RTM_GET; + rtm->rtm_flags = RTF_GATEWAY; + rtm->rtm_version = RTM_VERSION; + rtm->rtm_seq = seq; + rtm->rtm_addrs = RTA_DST | RTA_IFP; + rtm->rtm_pid = getpid(); + + int fd = socket(PF_ROUTE, SOCK_RAW, 0); + assert(fd > 0); + if (!write(fd, (char *) rtm, sizeof(buf))) { + log_error("get_gateway-bsd", "unable to send route request"); + return EXIT_FAILURE; + } + + size_t len; + while (rtm->rtm_type == RTM_GET && (len = read(fd, rtm, sizeof(buf))) > 0) { + if (len < (int) sizeof(*rtm)) { + return (-1); + } + if (rtm->rtm_type == RTM_GET && rtm->rtm_pid == getpid() && + rtm->rtm_seq == seq) { + if (rtm->rtm_errno) { + errno = rtm->rtm_errno; + return (-1); + } + break; + } + } + + struct sockaddr *sa = (struct sockaddr *) (rtm + 1); + for (int i = 0; i < RTAX_MAX; i++) { + if (rtm->rtm_addrs & (1 << i)) { + if ((1 << i) == RTA_IFP) { + struct sockaddr_dl *sdl = (struct sockaddr_dl *) sa; + if (!sdl) { + log_error("get_gateway-bsd", "unable to retrieve gateway"); + return EXIT_FAILURE; + } + char *_iface = xmalloc(sdl->sdl_nlen + 1); + memcpy(_iface, sdl->sdl_data, sdl->sdl_nlen); + _iface[sdl->sdl_nlen + 1] = 0; + *iface = _iface; + } + if ((1 << i) == RTA_GATEWAY) { + struct sockaddr_in *sin = (struct sockaddr_in *) sa; + int j; + for (j = 0; j < 4; j++) + gw_ip[j] = ((uint8_t *) &(sin->sin_addr.s_addr))[j]; + } + // next element + sa = (struct sockaddr *) (ROUNDUP(sa->sa_len) + (char *) sa); + } + } + close(fd); + + return EXIT_SUCCESS; +} + +char *get_default_iface(void) { + uint8_t t[16]; + char * retv = NULL; + _get_default_gw(t, &retv); + + return retv; +} + +int get_iface_ip(const char *iface_, uint8_t *ip, int ipv46_flag) { + assert(iface_); + + const char *iface = iface_; + if (!strncmp(iface_, ZCPREFIX, ZCPREFIX_LEN)) iface = iface_ + ZCPREFIX_LEN; + + struct ifaddrs *ifaddr, *ifa; + if (getifaddrs(&ifaddr)) { + log_error( + "get_gateway-linux", + "unable able to retrieve IPv%d list of network interfaces %s: %s", + ipv46_flag, iface, strerror(errno)); + return EXIT_FAILURE; + } + int af = 0; + if (ipv46_flag == IPV4_FLAG) + af = AF_INET; + else if (ipv46_flag == IPV6_FLAG) + af = AF_INET6; + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL || ifa->ifa_addr->sa_family != af) { + continue; + } + if (!strcmp(iface, ifa->ifa_name)) { + switch (ipv46_flag) { + case IPV4_FLAG: { + struct sockaddr_in *sin = (struct sockaddr_in *) ifa->ifa_addr; + int j; + for (j = 0; j < 4; j++) + ip[j] = ((uint8_t *) &(sin->sin_addr.s_addr))[j]; + return EXIT_SUCCESS; + } + case IPV6_FLAG: { + struct sockaddr_in6 *sin6 = + (struct sockaddr_in6 *) ifa->ifa_addr; + uint8_t *ip6_addr = (uint8_t *) &(sin6->sin6_addr); + if ((ip6_addr[0] >> 5u) != 0x01) continue; // global unicast + int j; + for (j = 0; j < 16; j++) + ip[j] = ip6_addr[j]; + return EXIT_SUCCESS; + } + default: + break; + } + } + } + + return EXIT_FAILURE; +} + +int get_iface_hw_addr(const char *iface_, unsigned char *hw_mac) { + const char *iface = iface_; + if (!strncmp(iface_, ZCPREFIX, ZCPREFIX_LEN)) iface = iface_ + ZCPREFIX_LEN; + + eth_t *e = eth_open(iface); + if (e) { + eth_addr_t eth_addr; + int res = eth_get(e, ð_addr); + if (res == 0) { + memcpy(hw_mac, eth_addr.data, ETHER_ADDR_LEN); + return EXIT_SUCCESS; + } + } + + return EXIT_FAILURE; +} + +// TODO: not support to get default gw's ipv6 address +int get_default_gw_ip(uint8_t *gw_ip, const char *iface) { + char *_iface = NULL; + _get_default_gw(gw_ip, &_iface); + if (strcmp(iface, _iface) != 0) { + log_error("get_gateway-bsd", + "interface specified (%s) does not match the interface of " + "the default gateway (%s). You will need to manually specify " + "the MAC address of your gateway.", + iface, _iface); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int get_hw_addr(const uint8_t *gw_ip_, UNUSED const char *iface, + unsigned char *hw_mac) { + struct in_addr * gw_ip = (struct in_addr *) gw_ip_; + arp_t * arp; + struct arp_entry entry; + + if (!gw_ip || !hw_mac) { + return EXIT_FAILURE; + } + + if ((arp = arp_open()) == NULL) { + log_error("get_gateway-bsd", "failed to open arp table"); + return EXIT_FAILURE; + } + + // Convert gateway ip to dnet struct format + memset(&entry, 0, sizeof(struct arp_entry)); + entry.arp_pa.addr_type = ADDR_TYPE_IP; + entry.arp_pa.addr_bits = IP_ADDR_BITS; + entry.arp_pa.addr_ip = gw_ip->s_addr; + + if (arp_get(arp, &entry) < 0) { + log_error("get_gateway-bsd", "failed to fetch arp entry"); + return EXIT_FAILURE; + } else { + memcpy(hw_mac, &entry.arp_ha.addr_eth, ETHER_ADDR_LEN); + } + arp_close(arp); + + return EXIT_SUCCESS; +} + +#endif // XMAP_GET_GATEWAY_BSD_H diff --git a/src/get_gateway-linux.h b/src/get_gateway-linux.h new file mode 100644 index 0000000..3182043 --- /dev/null +++ b/src/get_gateway-linux.h @@ -0,0 +1,345 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_GET_GATEWAY_LINUX_H +#define XMAP_GET_GATEWAY_LINUX_H + +#ifdef XMAP_GET_GATEWAY_BSD_H +#error "Don't include both get_gateway-bsd.h and get_gateway-linux.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#define GW_BUFFER_SIZE 64000 +#define ZCPREFIX "zc:" +#define ZCPREFIX_LEN 3 + +char *get_default_iface(void) { + char errbuf[PCAP_ERRBUF_SIZE]; + char *iface = pcap_lookupdev(errbuf); + if (iface == NULL) { + log_error("get_gateway-linux", "could not detect default network interface " + "(e.g. eth0). Try running as root or setting" + " interface using -i flag."); + } + return iface; +} + +int get_iface_ip(const char *iface_, uint8_t *ip, int ipv46_flag) { + assert(iface_); + + const char *iface = iface_; + if (!strncmp(iface_, ZCPREFIX, ZCPREFIX_LEN)) + iface = iface_ + ZCPREFIX_LEN; + + struct ifaddrs *ifaddr, *ifa; + if (getifaddrs(&ifaddr)) { + log_error("get_gateway-bsd", + "unable able to retrieve IPv%d list of network interfaces %s: %s", + ipv46_flag, iface, strerror(errno)); + return EXIT_FAILURE; + } + int af = 0; + if (ipv46_flag == IPV4_FLAG) + af = AF_INET; + else if (ipv46_flag == IPV6_FLAG) + af = AF_INET6; + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL || ifa->ifa_addr->sa_family != af) { + continue; + } + if (!strcmp(iface, ifa->ifa_name)) { + switch (ipv46_flag) { + case IPV4_FLAG: { + struct sockaddr_in *sin = (struct sockaddr_in *) ifa->ifa_addr; + int j; + for (j = 0; j < 4; j++) + ip[j] = ((uint8_t *) &(sin->sin_addr.s_addr))[j]; + return EXIT_SUCCESS; + } + case IPV6_FLAG: { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) ifa->ifa_addr; + uint8_t *ip6_addr = (uint8_t *) &(sin6->sin6_addr); + if ((ip6_addr[0] >> 5u) != 0x01) + continue; // global unicast + int j; + for (j = 0; j < 16; j++) + ip[j] = ip6_addr[j]; + return EXIT_SUCCESS; + } + default: + break; + } + } + } + return EXIT_FAILURE; +} + +int get_iface_hw_addr(const char *iface_, unsigned char *hw_mac) { + const char *iface = iface_; + if (!strncmp(iface_, ZCPREFIX, ZCPREFIX_LEN)) + iface = iface_ + ZCPREFIX_LEN; + + int s; + struct ifreq buffer; + + // Load the hwaddr from a dummy socket + s = socket(PF_INET, SOCK_DGRAM, 0); + if (s < 0) { + log_error("get_gateway-linux", "Unable to open socket: %s", + strerror(errno)); + return EXIT_FAILURE; + } + memset(&buffer, 0, sizeof(buffer)); + strncpy(buffer.ifr_name, iface, IFNAMSIZ); + ioctl(s, SIOCGIFHWADDR, &buffer); + close(s); + memcpy(hw_mac, buffer.ifr_hwaddr.sa_data, 6); + return EXIT_SUCCESS; +} + +int read_nl_sock(int sock, char *buf, int buf_len) { + int msg_len = 0; + char *pbuf = buf; + do { + int len = recv(sock, pbuf, buf_len - msg_len, 0); + if (len <= 0) { + log_error("get_gateway-linux", "recv failed: %s", strerror(errno)); + return -1; + } + struct nlmsghdr *nlhdr = (struct nlmsghdr *) pbuf; + if (NLMSG_OK(nlhdr, ((unsigned int) len)) == 0 || + nlhdr->nlmsg_type == NLMSG_ERROR) { + log_error("get_gateway-linux", "recv failed: %s", strerror(errno)); + return -1; + } + if (nlhdr->nlmsg_type == NLMSG_DONE) { + break; + } else { + msg_len += len; + pbuf += len; + } + if ((nlhdr->nlmsg_flags & NLM_F_MULTI) == 0) { + break; + } + } while (1); + return msg_len; +} + +int send_nl_req(uint16_t msg_type, uint32_t seq, void *payload, + uint32_t payload_len) { + int sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE); + if (sock < 0) { + log_error("get_gateway-linux", "unable to get socket: %s", strerror(errno)); + return -1; + } + if (NLMSG_SPACE(payload_len) < payload_len) { + close(sock); + // Integer overflow + return -1; + } + struct nlmsghdr *nlmsg; + nlmsg = xmalloc(NLMSG_SPACE(payload_len)); + + memset(nlmsg, 0, NLMSG_SPACE(payload_len)); + memcpy(NLMSG_DATA(nlmsg), payload, payload_len); + nlmsg->nlmsg_type = msg_type; + nlmsg->nlmsg_len = NLMSG_LENGTH(payload_len); + nlmsg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST; + nlmsg->nlmsg_seq = seq; + nlmsg->nlmsg_pid = getpid(); + + if (send(sock, nlmsg, nlmsg->nlmsg_len, 0) < 0) { + log_error("get_gateway-linux", "failure sending: %s", strerror(errno)); + return -1; + } + free(nlmsg); + return sock; +} + +// gw and iface[IF_NAMESIZE] MUST be allocated +static int _get_default_gw(uint8_t *gw_ip, char *iface) { + struct rtmsg req; + unsigned int nl_len; + char buf[8192]; + struct nlmsghdr *nlhdr; + + if (!gw_ip || !iface) { + return -1; + } + + // Send RTM_GETROUTE request + memset(&req, 0, sizeof(req)); + int sock = send_nl_req(RTM_GETROUTE, 0, &req, sizeof(req)); + + // Read responses + nl_len = read_nl_sock(sock, buf, sizeof(buf)); + if (nl_len <= 0) { + return -1; + } + + // Parse responses + nlhdr = (struct nlmsghdr *) buf; + while (NLMSG_OK(nlhdr, nl_len)) { + struct rtattr *rt_attr; + struct rtmsg *rt_msg; + int rt_len; + int has_gw = 0; + + rt_msg = (struct rtmsg *) NLMSG_DATA(nlhdr); + + // There could be multiple routing tables. Loop until we find the + // correct one. + if ((rt_msg->rtm_family != AF_INET) || + (rt_msg->rtm_table != RT_TABLE_MAIN)) { + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + continue; + } + + rt_attr = (struct rtattr *) RTM_RTA(rt_msg); + rt_len = RTM_PAYLOAD(nlhdr); + while (RTA_OK(rt_attr, rt_len)) { + switch (rt_attr->rta_type) { + case RTA_OIF: + if_indextoname(*(int *) RTA_DATA(rt_attr), iface); + break; + case RTA_GATEWAY: { + int j; + for (j = 0; j < 4; j++) + gw_ip[j] = ((unsigned char *) RTA_DATA(rt_attr))[j]; + has_gw = 1; + break; + } + } + rt_attr = RTA_NEXT(rt_attr, rt_len); + } + + if (has_gw) { + return 0; + } + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + } + return -1; +} + +int get_default_gw_ip(uint8_t *gw_ip, const char *iface) { + char _iface[IF_NAMESIZE]; + memset(_iface, 0, IF_NAMESIZE); + _get_default_gw(gw_ip, _iface); + if (strcmp(iface, _iface) != 0) { + log_error("get_gateway-linux", + "interface specified (%s) does not match " + "the interface of the default gateway (%s). You will need " + "to manually specify the MAC address of your gateway.", + iface, _iface); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int get_hw_addr(const uint8_t *gw_ip, const char *iface, unsigned char *hw_mac) { + struct ndmsg req; + memset(&req, 0, sizeof(struct ndmsg)); + + if (!gw_ip || !hw_mac) { + return EXIT_FAILURE; + } + // Send RTM_GETNEIGH request + req.ndm_family = AF_INET; + req.ndm_ifindex = if_nametoindex(iface); + req.ndm_state = NUD_REACHABLE; + req.ndm_type = NDA_LLADDR; + + int sock = send_nl_req(RTM_GETNEIGH, 1, &req, sizeof(req)); + // Read responses + char *buf = xmalloc(GW_BUFFER_SIZE); + int nl_len = read_nl_sock(sock, buf, GW_BUFFER_SIZE); + if (nl_len <= 0) { + free(buf); + return EXIT_FAILURE; + } + // Parse responses + struct nlmsghdr *nlhdr = (struct nlmsghdr *) buf; + while (NLMSG_OK(nlhdr, nl_len)) { + struct rtattr *rt_attr; + struct rtmsg *rt_msg; + int rt_len; + unsigned char mac[6]; + struct in_addr dst_ip; + int correct_ip = 0; + + rt_msg = (struct rtmsg *) NLMSG_DATA(nlhdr); + if ((rt_msg->rtm_family != AF_INET)) { + free(buf); + return EXIT_FAILURE; + } + rt_attr = (struct rtattr *) RTM_RTA(rt_msg); + rt_len = RTM_PAYLOAD(nlhdr); + while (RTA_OK(rt_attr, rt_len)) { + switch (rt_attr->rta_type) { + case NDA_LLADDR: + if (RTA_PAYLOAD(rt_attr) != IFHWADDRLEN) { + // could be using a VPN + log_error("get_gateway-linux", + "Unexpected hardware address length (%d)." + " If you are using a VPN, supply the --iplayer flag (and " + "provide an" + " interface via -i)", + RTA_PAYLOAD(rt_attr)); + return EXIT_FAILURE; + } + memcpy(mac, RTA_DATA(rt_attr), IFHWADDRLEN); + break; + case NDA_DST: + if (RTA_PAYLOAD(rt_attr) != sizeof(dst_ip)) { + // could be using a VPN + log_error("get_gateway-linux", + "Unexpected IP address length (%d)." + " If you are using a VPN, supply the --iplayer flag" + " (and provide an interface via -i)", + RTA_PAYLOAD(rt_attr)); + return EXIT_FAILURE; + } + memcpy(&dst_ip, RTA_DATA(rt_attr), sizeof(dst_ip)); + if (memcmp(&dst_ip, gw_ip, sizeof(dst_ip)) == 0) { + correct_ip = 1; + } + break; + } + rt_attr = RTA_NEXT(rt_attr, rt_len); + } + if (correct_ip) { + memcpy(hw_mac, mac, IFHWADDRLEN); + free(buf); + return EXIT_SUCCESS; + } + nlhdr = NLMSG_NEXT(nlhdr, nl_len); + } + free(buf); + return EXIT_FAILURE; +} + +#endif //XMAP_GET_GATEWAY_LINUX_H diff --git a/src/get_gateway.c b/src/get_gateway.c new file mode 100644 index 0000000..7b04194 --- /dev/null +++ b/src/get_gateway.c @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "get_gateway.h" + +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) + +#include "get_gateway-bsd.h" + +#else // (linux) +#include "get_gateway-linux.h" +#endif diff --git a/src/get_gateway.h b/src/get_gateway.h new file mode 100644 index 0000000..6a27fae --- /dev/null +++ b/src/get_gateway.h @@ -0,0 +1,29 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_GET_GATEWAY_H +#define XMAP_GET_GATEWAY_H + +#include "../lib/includes.h" +#include + +char *get_default_iface(void); + +int get_iface_ip(const char *iface, uint8_t *ip, int ipv46_flag); + +int get_iface_hw_addr(const char *iface, unsigned char *hw_mac); + +int get_default_gw_ip(uint8_t *gw_ip, const char *iface); + +int get_hw_addr(const uint8_t *gw_ip_, const char *iface, + unsigned char *hw_mac); + +#endif // XMAP_GET_GATEWAY_H diff --git a/src/iid_modules/iid_modules.c b/src/iid_modules/iid_modules.c new file mode 100644 index 0000000..3bb4953 --- /dev/null +++ b/src/iid_modules/iid_modules.c @@ -0,0 +1,44 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include + +extern iid_module_t module_zero; +extern iid_module_t module_low; +extern iid_module_t module_full; +extern iid_module_t module_rand; +extern iid_module_t module_set; +extern iid_module_t module_low_fill; +// Add your module here + +iid_module_t *iid_modules[] = { + &module_full, &module_low, &module_low_fill, + &module_rand, &module_set, &module_zero, + // Add your module here +}; + +iid_module_t *get_iid_module_by_name(const char *name) { + int len = (int) (sizeof(iid_modules) / sizeof(iid_modules[0])); + for (int i = 0; i < len; i++) { + if (!strcmp(iid_modules[i]->name, name)) { + return iid_modules[i]; + } + } + + return NULL; +} + +void print_iid_modules(void) { + int len = (int) (sizeof(iid_modules) / sizeof(iid_modules[0])); + for (int i = 0; i < len; i++) { + printf("%s\n", iid_modules[i]->name); + } +} diff --git a/src/iid_modules/iid_modules.h b/src/iid_modules/iid_modules.h new file mode 100644 index 0000000..c8e044f --- /dev/null +++ b/src/iid_modules/iid_modules.h @@ -0,0 +1,44 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_IID_MODULES_H +#define XMAP_IID_MODULES_H + +#include "../state.h" + +#define IP_MAX_BYTES 16 + +uint8_t IID[IP_MAX_BYTES]; // both for ipv4 & ipv6 + +// called at sender initialization +typedef int (*iid_global_init_cb)(struct state_conf *conf); + +// called at sender thread initialization +typedef int (*iid_thread_init_cb)(void); + +// called at sender thread getting IP suffix +typedef int (*iid_get_cb)(void *iid, int iid_num, void *args); + +// called at the end of scan +typedef int (*iid_close_cb)(void); + +typedef struct iid_module { + const char * name; + iid_global_init_cb global_init; + iid_thread_init_cb thread_init; + iid_get_cb get_current_iid; + iid_close_cb close; + const char * helptext; +} iid_module_t; + +iid_module_t *get_iid_module_by_name(const char *name); + +void print_iid_modules(void); + +#endif // XMAP_IID_MODULES_H diff --git a/src/iid_modules/module_full.c b/src/iid_modules/module_full.c new file mode 100644 index 0000000..8c5f616 --- /dev/null +++ b/src/iid_modules/module_full.c @@ -0,0 +1,51 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include +#include + +iid_module_t module_full; +static int ipv46_bytes; + +int full_global_init(struct state_conf *conf) { + assert(conf); + ipv46_bytes = conf->ipv46_bytes; + memset(IID, 0, ipv46_bytes); + + int i; + for (i = xconf.max_probe_len / 8; i < ipv46_bytes; i++) + IID[i] = 0xff; + i = xconf.max_probe_len % 8; + if (i) { + IID[xconf.max_probe_len / 8] >>= i; + } + + return EXIT_SUCCESS; +} + +int full_thread_init(void) { return EXIT_SUCCESS; } + +int full_get_current_iid(void *iid, UNUSED int iid_index, UNUSED void *args) { + memcpy(iid, IID, ipv46_bytes); + + return EXIT_SUCCESS; +} + +int full_close(void) { return EXIT_SUCCESS; } + +iid_module_t module_full = {.name = "full", + .global_init = full_global_init, + .thread_init = full_thread_init, + .get_current_iid = full_get_current_iid, + .close = full_close, + .helptext = + "Full mode IID (suffix), e.g., " + "2001:db8:1234:5678:FFFF:FFFF:FFFF:FFFF."}; diff --git a/src/iid_modules/module_low.c b/src/iid_modules/module_low.c new file mode 100644 index 0000000..46a2b9a --- /dev/null +++ b/src/iid_modules/module_low.c @@ -0,0 +1,52 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include +#include + +iid_module_t module_low; +static int ipv46_bytes; + +int low_global_init(struct state_conf *conf) { + assert(conf); + ipv46_bytes = conf->ipv46_bytes; + memset(IID, 0, ipv46_bytes); + ((uint8_t *) IID)[ipv46_bytes - 1] = 1; + + int i = 0; + for (i = 0; i < xconf.max_probe_len / 8; i++) + IID[i] = 0x00; + i = xconf.max_probe_len % 8; + if (i) { + int j = 0xff >> i; + IID[xconf.max_probe_len / 8] &= j; + } + + return EXIT_SUCCESS; +} + +int low_thread_init(void) { return EXIT_SUCCESS; } + +int low_get_current_iid(void *iid, UNUSED int iid_index, UNUSED void *args) { + memcpy(iid, IID, ipv46_bytes); + + return EXIT_SUCCESS; +} + +int low_close(void) { return EXIT_SUCCESS; } + +iid_module_t module_low = { + .name = "low", + .global_init = low_global_init, + .thread_init = low_thread_init, + .get_current_iid = low_get_current_iid, + .close = low_close, + .helptext = "Low mode IID (suffix), e.g., 2001:db8:1234:5678::1."}; diff --git a/src/iid_modules/module_low_fill.c b/src/iid_modules/module_low_fill.c new file mode 100644 index 0000000..d32a88d --- /dev/null +++ b/src/iid_modules/module_low_fill.c @@ -0,0 +1,53 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include +#include + +iid_module_t module_low_fill; +static int ipv46_bytes; + +int low_fill_global_init(struct state_conf *conf) { + assert(conf); + ipv46_bytes = conf->ipv46_bytes; + memset(IID, 0, ipv46_bytes); + + int i = (xconf.max_probe_len / 8) > (ipv46_bytes - 2) + ? xconf.max_probe_len / 8 + : ipv46_bytes - 2; + for (; i < ipv46_bytes; i++) + IID[i] = 0xff; + i = xconf.max_probe_len % 8; + if (i) { + IID[xconf.max_probe_len / 8] >>= i; + } + + return EXIT_SUCCESS; +} + +int low_fill_thread_init(void) { return EXIT_SUCCESS; } + +int low_fill_get_current_iid(void *iid, UNUSED int iid_index, + UNUSED void *args) { + memcpy(iid, IID, ipv46_bytes); + + return EXIT_SUCCESS; +} + +int low_fill_close(void) { return EXIT_SUCCESS; } + +iid_module_t module_low_fill = { + .name = "low_fill", + .global_init = low_fill_global_init, + .thread_init = low_fill_thread_init, + .get_current_iid = low_fill_get_current_iid, + .close = low_fill_close, + .helptext = "Low fill mode IID (suffix), e.g., 2001:db8:1234:5678::FFFF."}; diff --git a/src/iid_modules/module_rand.c b/src/iid_modules/module_rand.c new file mode 100644 index 0000000..ceddf8d --- /dev/null +++ b/src/iid_modules/module_rand.c @@ -0,0 +1,67 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include +#include + +#include "../../lib/gmp-ext.h" + +iid_module_t module_rand; + +static gmp_randstate_t random_state; +static int ipv46_bytes; +static int iid_bits; +static int max_probe_len_div8; +static int max_probe_len_mod8; + +int rand_global_init(struct state_conf *conf) { + assert(conf); + gmp_randinit_default(random_state); + time_t t = time(NULL); + gmp_randseed_ui(random_state, t); + ipv46_bytes = conf->ipv46_bytes; + iid_bits = xconf.ipv46_bits - xconf.max_probe_len; + max_probe_len_div8 = xconf.max_probe_len / 8; + max_probe_len_mod8 = xconf.max_probe_len % 8; + + return EXIT_SUCCESS; +} + +int rand_thread_init(void) { return EXIT_SUCCESS; } + +int rand_get_current_iid(void *iid, UNUSED int iid_index, UNUSED void *args) { + mpz_t rand_m; + mpz_init(rand_m); + mpz_urandomb(rand_m, random_state, iid_bits); + mpz_to_uint8s(rand_m, (uint8_t *) iid, ipv46_bytes); + uint8_t *IID = (uint8_t *) iid; + + for (int i = 0; i < max_probe_len_div8; i++) + IID[i] = 0x00; + if (max_probe_len_mod8) { + int j = 0xff >> max_probe_len_mod8; + IID[max_probe_len_div8] &= j; + } + mpz_clear(rand_m); + + return EXIT_SUCCESS; +} + +int rand_close_(void) { return EXIT_SUCCESS; } + +iid_module_t module_rand = {.name = "rand", + .global_init = rand_global_init, + .thread_init = rand_thread_init, + .get_current_iid = rand_get_current_iid, + .close = rand_close_, + .helptext = + "Random mode IID (suffix), e.g., " + "2001:db8:1234:5678:1783:ab42:9247:cb38."}; diff --git a/src/iid_modules/module_set.c b/src/iid_modules/module_set.c new file mode 100644 index 0000000..172f39d --- /dev/null +++ b/src/iid_modules/module_set.c @@ -0,0 +1,65 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include +#include + +#include "../../lib/logger.h" +#include "../../lib/util.h" + +iid_module_t module_set; +static int ipv46_bytes; + +int set_global_init(struct state_conf *conf) { + assert(conf); + ipv46_bytes = conf->ipv46_bytes; + memset(IID, 0, ipv46_bytes); + + if (!xconf.iid_args) { + log_fatal("iid-module", "Set mode: NULL args for IPv%d", + xconf.ipv46_flag); + } + if (!inet_str2in(xconf.iid_args, IID, xconf.ipv46_flag)) { + log_fatal("iid-module", "Set mode: unknown args `%s' for IPv%d", + xconf.iid_args, xconf.ipv46_flag); + } + + int i; + for (i = 0; i < xconf.max_probe_len / 8; i++) + IID[i] = 0x00; + i = xconf.max_probe_len % 8; + if (i) { + int j = 0xff >> i; + IID[xconf.max_probe_len / 8] &= j; + } + + return EXIT_SUCCESS; +} + +int set_thread_init(void) { return EXIT_SUCCESS; } + +int set_get_current_iid(void *iid, UNUSED int iid_index, UNUSED void *args) { + memcpy(iid, IID, ipv46_bytes); + + return EXIT_SUCCESS; +} + +int set_close(void) { return EXIT_SUCCESS; } + +iid_module_t module_set = {.name = "set", + .global_init = set_global_init, + .thread_init = set_thread_init, + .get_current_iid = set_get_current_iid, + .close = set_close, + .helptext = + "Set mode IID (suffix), extract the suffix of " + "`--iid-args' as targets' IID (suffix).\n" + " --iid-args=ipv46_address."}; diff --git a/src/iid_modules/module_zero.c b/src/iid_modules/module_zero.c new file mode 100644 index 0000000..819ddf5 --- /dev/null +++ b/src/iid_modules/module_zero.c @@ -0,0 +1,42 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iid_modules.h" + +#include +#include + +iid_module_t module_zero; +static int ipv46_bytes; + +int zero_global_init(struct state_conf *conf) { + assert(conf); + ipv46_bytes = conf->ipv46_bytes; + memset(IID, 0, ipv46_bytes); + + return EXIT_SUCCESS; +} + +int zero_thread_init(void) { return EXIT_SUCCESS; } + +int zero_get_current_iid(void *iid, UNUSED int iid_index, UNUSED void *args) { + memcpy(iid, IID, ipv46_bytes); + + return EXIT_SUCCESS; +} + +int zero_close(void) { return EXIT_SUCCESS; } + +iid_module_t module_zero = { + .name = "zero", + .global_init = zero_global_init, + .thread_init = zero_thread_init, + .get_current_iid = zero_get_current_iid, + .close = zero_close, + .helptext = "Zero mode IID (suffix), e.g., 2001:db8:1234:5678::."}; diff --git a/src/ip_target_file.c b/src/ip_target_file.c new file mode 100644 index 0000000..1b20939 --- /dev/null +++ b/src/ip_target_file.c @@ -0,0 +1,135 @@ +/* + * ZMapv6 Copyright 2016 Chair of Network Architectures and Services + * Technical University of Munich + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "ip_target_file.h" + +#include +#include +#include +#include +#include + +#include "state.h" + +#include "../lib/blocklist.h" +#include "../lib/logger.h" +#include "../lib/util.h" + +#define LOGGER_NAME "ip_target_file" + +static int ipv46_flag; + +int64_t ip_target_file_init(char *file) { + if (!file) + log_fatal(LOGGER_NAME, "ip_target_file_init called with NULL filename"); + ipv46_flag = xconf.ipv46_flag; + + return get_file_lines(file); +} + +int ip_target_set_thread_pos(iterator_t *it) { + FILE *fp = fopen(xconf.list_of_ips_filename, "r"); + if (fp == NULL) { + log_fatal(LOGGER_NAME, + "ip_target_set_thread_pos called with NULL filename"); + } + + char line[64]; + long pos; + int i = 0; + uint64_t count = 0; + shard_t *sd = get_shard(it, i); + uint8_t num_threads = get_num_threads(it); + do { + if (sd->ip_target_file_params.first == count) { + fgetpos(fp, (fpos_t *) &pos); + rewind(sd->ip_target_file_params.fp); + fseek(sd->ip_target_file_params.fp, pos, SEEK_SET); + sd->ip_target_file_params.pos = pos; + i++; + if (i >= num_threads) return EXIT_SUCCESS; + sd = get_shard(it, i); + } + count++; + } while (fgets(line, sizeof(line), fp)); + fclose(fp); + + return EXIT_SUCCESS; +} + +static int recover_thread_file_params(shard_t *shard) { + if (shard->ip_target_file_params.port_current + 1 >= + shard->ip_target_file_params.port_total) + return EXIT_FAILURE; + shard->ip_target_file_params.port_current++; + shard->ip_target_file_params.current = shard->ip_target_file_params.first; + rewind(shard->ip_target_file_params.fp); + fseek(shard->ip_target_file_params.fp, shard->ip_target_file_params.pos, + SEEK_SET); + return EXIT_SUCCESS; +} + +int ip_target_file_get_ip(void *ip, shard_t *shard) { + if (shard->ip_target_file_params.current >= + shard->ip_target_file_params.last) + if (recover_thread_file_params(shard)) return EXIT_FAILURE; + + FILE *fp = shard->ip_target_file_params.fp; + assert(fp); + + char line[64]; + char *ret, *pos; + + ret = fgets(line, sizeof(line), fp); + if (ret == NULL) + if (recover_thread_file_params(shard)) return EXIT_FAILURE; + + shard->ip_target_file_params.current++; + pos = strchr(line, '\n'); + if (pos != NULL) *pos = '\0'; + pos = strchr(line, '/'); + if (pos != NULL) *pos = '\0'; + if (!inet_str2in(line, ip, ipv46_flag)) { + if (!xconf.ignore_filelist_error) + log_fatal(LOGGER_NAME, + "could not parse IPv%d address from line: %s: %s", + ipv46_flag, line, strerror(errno)); + goto goon; + } + + while (!blocklist_is_allowed_ip(ip)) { + goon: + if (shard->ip_target_file_params.current >= + shard->ip_target_file_params.last) + if (recover_thread_file_params(shard)) return EXIT_FAILURE; + ret = fgets(line, sizeof(line), fp); + if (ret == NULL) + if (recover_thread_file_params(shard)) return EXIT_FAILURE; + shard->ip_target_file_params.current++; + pos = strchr(line, '\n'); + if (pos != NULL) *pos = '\0'; + pos = strchr(line, '/'); + if (pos != NULL) *pos = '\0'; + if (!inet_str2in(line, ip, ipv46_flag)) { + if (!xconf.ignore_filelist_error) + log_fatal(LOGGER_NAME, + "could not parse IPv%d address from line: %s: %s", + ipv46_flag, line, strerror(errno)); + } + } + + return EXIT_SUCCESS; +} + +port_h_t ip_target_file_get_port(shard_t *shard) { + return xconf.target_port_list[shard->ip_target_file_params.port_current]; +} diff --git a/src/ip_target_file.h b/src/ip_target_file.h new file mode 100644 index 0000000..24d4d13 --- /dev/null +++ b/src/ip_target_file.h @@ -0,0 +1,29 @@ +/* + * ZMapv6 Copyright 2016 Chair of Network Architectures and Services + * Technical University of Munich + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_IP_TARGET_FILE_H +#define XMAP_IP_TARGET_FILE_H + +#include +#include + +#include "iterator.h" + +int64_t ip_target_file_init(char *file); + +int ip_target_set_thread_pos(iterator_t *it); + +int ip_target_file_get_ip(void *ip, shard_t *shard); + +port_h_t ip_target_file_get_port(shard_t *shard); + +#endif // XMAP_IP_TARGET_FILE_H diff --git a/src/iterator.c b/src/iterator.c new file mode 100644 index 0000000..c40e289 --- /dev/null +++ b/src/iterator.c @@ -0,0 +1,173 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "iterator.h" + +#include +#include + +#include "cyclic.h" +#include "ip_target_file.h" +#include "state.h" + +#include "../lib/blocklist.h" +#include "../lib/gmp-ext.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +struct iterator { + cycle_t cycle; + uint8_t num_threads; + shard_t * thread_shards; + uint8_t * complete; + pthread_mutex_t mutex; + uint32_t curr_threads; +}; + +// callback instance +void shard_complete_cbi(uint8_t thread_id, void *args) { + iterator_t *it = (iterator_t *) args; + assert(thread_id < it->num_threads); + + pthread_mutex_lock(&it->mutex); + it->complete[thread_id] = 1; + it->curr_threads--; + + shard_t *s = &it->thread_shards[thread_id]; + xsend.packets_sent += s->state.packets_sent; + xsend.hosts_scanned += s->state.hosts_scanned; + xsend.blocklisted += s->state.hosts_blocklisted; + xsend.allowlisted += s->state.hosts_allowlisted; + xsend.sendto_failures += s->state.packets_failed; + xsend.packets_tried += s->state.packets_tried; + + uint8_t done = 1; + for (uint8_t i = 0; done && (i < it->num_threads); i++) { + done = done && it->complete[i]; + } + if (done) { + xsend.finish = now(); + xsend.complete = 1; + mpz_set(xsend.first_scanned, it->thread_shards[0].state.first_scanned); + } + + pthread_mutex_unlock(&it->mutex); +} + +iterator_t *iterator_init(uint8_t num_threads, uint16_t shard, + uint16_t num_shards) { + log_debug("iterator", "iterator_init start"); + + mpz_t num_addrs_ports; + mpz_init(num_addrs_ports); + blocklist_count_allowed_ip_port(num_addrs_ports); + mpz_t group_min_size; + mpz_init_set(group_min_size, num_addrs_ports); + + mpz_t two, temp; + mpz_init_set_ui(two, 2); + mpz_init(temp); + + iterator_t * it = xmalloc(sizeof(struct iterator)); + const cyclic_group_t *group = get_group(group_min_size); + + mpz_pow_ui(temp, two, xconf.max_probe_port_len); + if (mpz_gt(num_addrs_ports, temp)) { + mpz_sub_ui(temp, temp, 1); + mpz_set(xsend.max_index, temp); + } else { + mpz_set(xsend.max_index, num_addrs_ports); + } + log_debug("iterator", "max index %s", mpz_to_str10(xsend.max_index)); + + it->cycle = make_cycle(group, xconf.aes); + it->num_threads = num_threads; + it->curr_threads = num_threads; + it->thread_shards = xcalloc(num_threads, sizeof(shard_t)); + it->complete = xcalloc(it->num_threads, sizeof(uint8_t)); + pthread_mutex_init(&it->mutex, NULL); + + for (uint8_t i = 0; i < num_threads; i++) { + shard_init(&it->thread_shards[i], shard, num_shards, i, num_threads, + xsend.max_targets, xsend.max_packets, xconf.list_of_ip_count, + &it->cycle, shard_complete_cbi, it); + } + + if (xconf.list_of_ips_filename) ip_target_set_thread_pos(it); + + mpz_set(xconf.generator, it->cycle.generator); + + log_debug("iterator", "iterator_init completed"); + + return it; +} + +uint64_t iterator_get_scanned(iterator_t *it) { + uint64_t sent = 0; + for (uint8_t i = 0; i < it->num_threads; i++) + sent += it->thread_shards[i].state.hosts_scanned; + + return sent; +} + +uint64_t iterator_get_sent(iterator_t *it) { + uint64_t sent = 0; + for (uint8_t i = 0; i < it->num_threads; i++) + sent += it->thread_shards[i].state.packets_sent; + + return sent; +} + +uint64_t iterator_get_iterations(iterator_t *it) { + uint64_t iterations = 0; + for (uint8_t i = 0; i < it->num_threads; i++) + iterations += it->thread_shards[i].iterations; + + return iterations; +} + +uint64_t iterator_get_fail(iterator_t *it) { + uint32_t fails = 0; + for (uint8_t i = 0; i < it->num_threads; i++) + fails += it->thread_shards[i].state.packets_failed; + + return fails; +} + +uint64_t iterator_get_tried(iterator_t *it) { + uint64_t sent = 0; + for (uint8_t i = 0; i < it->num_threads; i++) + sent += it->thread_shards[i].state.packets_tried; + + return sent; +} + +shard_t *get_shard(iterator_t *it, uint8_t thread_id) { + assert(thread_id < it->num_threads); + + return &it->thread_shards[thread_id]; +} + +uint8_t get_num_threads(iterator_t *it) { return it->num_threads; } + +uint32_t iterator_get_curr_send_threads(iterator_t *it) { + assert(it); + + return it->curr_threads; +} + +void iterator_free(iterator_t *it) { + for (int i = 0; i < it->num_threads; i++) + shard_free(&(it->thread_shards[i])); + close_cycle(it->cycle); + + log_debug("iterator", "cleaning up"); +} diff --git a/src/iterator.h b/src/iterator.h new file mode 100644 index 0000000..6660c64 --- /dev/null +++ b/src/iterator.h @@ -0,0 +1,43 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_ITERATOR_H +#define XMAP_ITERATOR_H + +#include +#include + +#include "shard.h" + +typedef struct iterator iterator_t; + +iterator_t *iterator_init(uint8_t num_threads, uint16_t shard, + uint16_t num_shards); + +uint64_t iterator_get_scanned(iterator_t *it); + +uint64_t iterator_get_sent(iterator_t *it); + +uint64_t iterator_get_iterations(iterator_t *it); + +uint64_t iterator_get_fail(iterator_t *it); + +uint64_t iterator_get_tried(iterator_t *it); + +shard_t *get_shard(iterator_t *it, uint8_t thread_id); + +uint8_t get_num_threads(iterator_t *it); + +uint32_t iterator_get_curr_send_threads(iterator_t *it); + +void iterator_free(iterator_t *it); + +#endif // XMAP_ITERATOR_H diff --git a/src/lexer.l b/src/lexer.l new file mode 100644 index 0000000..48e0ad2 --- /dev/null +++ b/src/lexer.l @@ -0,0 +1,36 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +%{ +#pragma GCC diagnostic ignored "-Wredundant-decls" +#pragma GCC diagnostic ignored "-Wmissing-noreturn" + +#include +#include "parser.h" + +%} + +%option noinput +%option nounput +%% +[0-9]+ yylval.int_literal = (uint64_t) atoll(yytext); return T_NUMBER; +\n /* Ignore end of line */ +[ \t]+ /* Ignore whitespace */ +!= return T_NOT_EQ; +>= return T_GT_EQ; +"<=" return T_LT_EQ; +&& return T_AND; +"||" return T_OR; += return '='; +">" return '>'; +"<" return '<'; +"(" return '('; +")" return ')'; +[a-zA-Z][-_a-zA-Z0-9]+ yylval.string_literal = strdup(yytext); return T_FIELD; + +%% diff --git a/src/monitor.c b/src/monitor.c new file mode 100644 index 0000000..ea98c3a --- /dev/null +++ b/src/monitor.c @@ -0,0 +1,517 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// module responsible for printing on-screen updates during the scan process + +#include "monitor.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iterator.h" +#include "probe_modules/probe_modules.h" +#include "recv.h" +#include "state.h" + +#include "../lib/gmp-ext.h" +#include "../lib/lockfd.h" +#include "../lib/logger.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#define UPDATE_INTERVAL 1 // seconds +#define NUMBER_STR_LEN 20 +#define WARMUP_PERIOD 5 +#define MIN_HITRATE_TIME_WINDOW 5 // seconds + +// internal monitor status that is used to track deltas +typedef struct internal_scan_status { + double last_now; + uint64_t last_sent; + uint64_t last_tried_sent; + uint64_t last_send_failures; + uint64_t last_recv_net_success; + uint64_t last_recv_app_success; + uint64_t last_recv_total; + uint64_t last_pcap_drop; + double min_hitrate_start; +} int_status_t; + +// exportable status information that can be printed to screen +typedef struct export_scan_status { + uint64_t total_sent; + uint64_t total_tried_sent; + uint64_t recv_success_unique; + uint64_t app_recv_success_unique; + uint64_t total_recv; + uint32_t complete; + uint32_t send_threads; + double percent_complete; + + double hitrate; // network, e.g. SYN-ACK vs RST + double app_hitrate; // application level, e.g. DNS response versus correct + // lookup. + + double send_rate; + char send_rate_str[NUMBER_STR_LEN]; + double send_rate_avg; + char send_rate_avg_str[NUMBER_STR_LEN]; + + double send_bandwidth; + char send_bandwidth_str[NUMBER_STR_LEN]; + double send_bandwidth_avg; + char send_bandwidth_avg_str[NUMBER_STR_LEN]; + + double recv_rate; + char recv_rate_str[NUMBER_STR_LEN]; + double recv_avg; + char recv_avg_str[NUMBER_STR_LEN]; + double recv_total_rate; + double recv_total_avg; + + double app_success_rate; + char app_success_rate_str[NUMBER_STR_LEN]; + double app_success_avg; + char app_success_avg_str[NUMBER_STR_LEN]; + + uint64_t pcap_drop; + uint64_t pcap_ifdrop; + uint64_t pcap_drop_total; + char pcap_drop_total_str[NUMBER_STR_LEN]; + double pcap_drop_last; + char pcap_drop_last_str[NUMBER_STR_LEN]; + double pcap_drop_avg; + char pcap_drop_avg_str[NUMBER_STR_LEN]; + + uint32_t time_remaining; + char time_remaining_str[NUMBER_STR_LEN]; + uint32_t time_past; + char time_past_str[NUMBER_STR_LEN]; + + uint64_t fail_total; + double fail_avg; + double fail_last; + float seconds_under_min_hitrate; + +} export_status_t; + +static FILE *status_fd = NULL; + +// find minimum of an array of doubles +static double min_d(double array[], int n) { + double value = INFINITY; + for (int i = 0; i < n; i++) { + if (array[i] < value) { + value = array[i]; + } + } + + return value; +} + +// estimate time remaining time based on config and state +double compute_remaining_time(double age, uint64_t hosts_scanned, + uint64_t packets_sent) { + if (!xsend.complete) { + double remaining[] = {INFINITY, INFINITY, INFINITY, + INFINITY, INFINITY, INFINITY}; + if (xconf.list_of_ips_filename) { + // Estimate progress using group iterations + double done = + (double) hosts_scanned / (xconf.list_of_ip_port_count); + remaining[0] = (1. - done) * (age / done) + xconf.cooldown_secs; + } + if (xsend.max_targets) { + double done = (double) hosts_scanned / + (double) (xsend.max_targets / xconf.total_shards); + remaining[1] = (1. - done) * (age / done) + xconf.cooldown_secs; + } + if (xsend.max_packets) { + double done = (double) packets_sent / + (double) (xsend.max_packets / xconf.total_shards); + remaining[2] = (1. - done) * (age / done) + xconf.cooldown_secs; + } + if (xconf.max_runtime) { + remaining[3] = (xconf.max_runtime - age) + xconf.cooldown_secs; + } + if (xconf.max_results) { + double done = (double) xrecv.filter_success / xconf.max_results; + remaining[4] = (1. - done) * (age / done); + } + if (mpz_not_zero(xconf.total_allowed_ip_port_actual)) { + mpf_t host_scanned_t, max_ip_port_num_t; + mpf_init_set_d(host_scanned_t, + (double) hosts_scanned * xconf.total_shards); + mpf_init(max_ip_port_num_t); + mpf_set_z(max_ip_port_num_t, xconf.total_allowed_ip_port_actual); + mpf_div(host_scanned_t, host_scanned_t, max_ip_port_num_t); + double done = mpf_get_d(host_scanned_t); + mpf_clear(host_scanned_t); + mpf_clear(max_ip_port_num_t); + remaining[5] = (1. - done) * (age / done) + xconf.cooldown_secs; + } + return min_d(remaining, sizeof(remaining) / sizeof(double)); + } else { + double remaining_secs = xconf.cooldown_secs - (now() - xsend.finish); + if (remaining_secs < 0) remaining_secs = 0.0; + return remaining_secs; + } +} + +static void update_pcap_stats(pthread_mutex_t *recv_ready_mutex) { + // ask pcap for fresh values + pthread_mutex_lock(recv_ready_mutex); + recv_update_stats(); + pthread_mutex_unlock(recv_ready_mutex); +} + +static void export_stats(int_status_t *intrnl, export_status_t *exp, + iterator_t *it) { + uint64_t total_scanned = iterator_get_scanned(it); + uint64_t total_sent = iterator_get_sent(it); + uint64_t total_tried = iterator_get_tried(it); + uint64_t total_fail = iterator_get_fail(it); + uint64_t total_recv = xrecv.pcap_recv; + uint64_t recv_success = xrecv.filter_success; + uint64_t app_success = xrecv.app_success_unique; + double cur_time = now(); + double age = cur_time - xsend.start; // time of entire scan + // time since the last time we updated + double delta = cur_time - intrnl->last_now; + double remaining_secs = + compute_remaining_time(age, total_scanned, total_sent); + + // export amount of time the scan has been running + if (age < WARMUP_PERIOD) { + exp->time_remaining_str[0] = '\0'; + } else { + char buf[20]; + time_string(ceil(remaining_secs), 1, buf, sizeof(buf)); + snprintf(exp->time_remaining_str, NUMBER_STR_LEN, " (%s left)", buf); + } + exp->time_past = age; + exp->time_remaining = remaining_secs; + time_string((int) age, 0, exp->time_past_str, NUMBER_STR_LEN); + + // export recv statistics + exp->recv_rate = + ceil((double) (recv_success - intrnl->last_recv_net_success) / delta); + number_string(exp->recv_rate, exp->recv_rate_str, NUMBER_STR_LEN); + exp->recv_avg = recv_success / age; + number_string(exp->recv_avg, exp->recv_avg_str, NUMBER_STR_LEN); + exp->recv_total_rate = + (double) (total_recv - intrnl->last_recv_total) / delta; + exp->recv_total_avg = total_recv / age; + + // application level statistics + if (xconf.fsconf.app_success_index >= 0) { + exp->app_success_rate = + (double) (app_success - intrnl->last_recv_app_success) / delta; + number_string(exp->app_success_rate, exp->app_success_rate_str, + NUMBER_STR_LEN); + exp->app_success_avg = (app_success / age); + number_string(exp->app_success_avg, exp->app_success_avg_str, + NUMBER_STR_LEN); + } + + if (!total_sent) { + exp->hitrate = 0; + exp->app_hitrate = 0; + } else { + exp->hitrate = recv_success * 100.0 / total_sent; + exp->app_hitrate = app_success * 100.0 / total_sent; + } + + if (age > WARMUP_PERIOD && exp->hitrate < xconf.min_hitrate) { + if (fabs(intrnl->min_hitrate_start) < .00001) { + intrnl->min_hitrate_start = cur_time; + } + } else { + intrnl->min_hitrate_start = 0.0; + } + if (fabs(intrnl->min_hitrate_start) < .00001) { + exp->seconds_under_min_hitrate = 0; + } else { + exp->seconds_under_min_hitrate = cur_time - intrnl->min_hitrate_start; + } + + if (!xsend.complete) { + exp->send_rate = + ceil((double) (total_sent - intrnl->last_sent) / delta); + number_string(exp->send_rate, exp->send_rate_str, NUMBER_STR_LEN); + exp->send_rate_avg = total_sent / age; + number_string(exp->send_rate_avg, exp->send_rate_avg_str, + NUMBER_STR_LEN); + + size_t pkt_len = (xconf.probe_module->packet_length > 60) + ? (xconf.probe_module->packet_length + 24) + : 84; + exp->send_bandwidth = exp->send_rate * pkt_len * 8; + bits_string(exp->send_bandwidth, exp->send_bandwidth_str, + NUMBER_STR_LEN); + exp->send_bandwidth_avg = exp->send_rate_avg * pkt_len * 8; + bits_string(exp->send_bandwidth_avg, exp->send_bandwidth_avg_str, + NUMBER_STR_LEN); + } else { + exp->send_rate_avg = total_sent / (xsend.finish - xsend.start); + number_string(exp->send_rate_avg, exp->send_rate_avg_str, + NUMBER_STR_LEN); + + size_t pkt_len = (xconf.probe_module->packet_length > 60) + ? (xconf.probe_module->packet_length + 24) + : 84; + exp->send_bandwidth_avg = exp->send_rate_avg * pkt_len * 8; + bits_string(exp->send_bandwidth_avg, exp->send_bandwidth_avg_str, + NUMBER_STR_LEN); + } + + // export other pre-calculated values + exp->total_sent = total_sent; + exp->total_tried_sent = total_tried; + exp->percent_complete = 100. * age / (age + remaining_secs); + exp->recv_success_unique = recv_success; + exp->app_recv_success_unique = app_success; + exp->total_recv = total_recv; + exp->complete = xsend.complete; + + // pcap dropped packets + exp->pcap_drop = xrecv.pcap_drop; + exp->pcap_ifdrop = xrecv.pcap_ifdrop; + exp->pcap_drop_total = exp->pcap_drop + exp->pcap_ifdrop; + exp->pcap_drop_last = + (double) (exp->pcap_drop_total - intrnl->last_pcap_drop) / delta; + exp->pcap_drop_avg = exp->pcap_drop_total / age; + number_string(exp->pcap_drop_total, exp->pcap_drop_total_str, + NUMBER_STR_LEN); + number_string(exp->pcap_drop_last, exp->pcap_drop_last_str, NUMBER_STR_LEN); + number_string(exp->pcap_drop_avg, exp->pcap_drop_avg_str, NUMBER_STR_LEN); + + xsend.sendto_failures = total_fail; + exp->fail_total = xsend.sendto_failures; + exp->fail_last = + (double) (exp->fail_total - intrnl->last_send_failures) / delta; + exp->fail_avg = exp->fail_total / age; + + // misc + exp->send_threads = iterator_get_curr_send_threads(it); + + // Update internal stats + intrnl->last_now = cur_time; + intrnl->last_sent = exp->total_sent; + intrnl->last_recv_net_success = exp->recv_success_unique; + intrnl->last_recv_app_success = exp->app_recv_success_unique; + intrnl->last_pcap_drop = exp->pcap_drop_total; + intrnl->last_send_failures = exp->fail_total; + intrnl->last_recv_total = exp->total_recv; +} + +static void log_drop_warnings(export_status_t *exp) { + if (exp->pcap_drop_last / exp->recv_rate > 0.05) { + log_warn("monitor", + "Dropped %.0f packets in the last second, (%u total dropped " + "(pcap: %u + iface: %u))", + exp->pcap_drop_last, exp->pcap_drop_total, exp->pcap_drop, + exp->pcap_ifdrop); + } + if (exp->fail_last / exp->send_rate > 0.01) { + log_warn("monitor", + "Failed to send %.0f packets/sec (%u total failures)", + exp->fail_last, exp->fail_total); + } +} + +static void onscreen_appsuccess(export_status_t *exp) { + // this when probe module handles application-level success rates + if (!exp->complete) { + fprintf(stderr, + "%5s %0.0f%%%s; " + "sent: %llu %sp/s %sb/s (%sp/s %sb/s avg); " + "recv: %llu %sp/s (%sp/s avg); " + "app success: %llu %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%% " + "app hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, exp->send_rate_str, + exp->send_bandwidth_str, exp->send_rate_avg_str, + exp->send_bandwidth_avg_str, exp->recv_success_unique, + exp->recv_rate_str, exp->recv_avg_str, + exp->app_recv_success_unique, exp->app_success_rate_str, + exp->app_success_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate, exp->app_hitrate); + } else { + fprintf(stderr, + "%5s %0.0f%%%s; " + "sent: %llu done (%sp/s %sb/s avg); " + "recv: %llu %sp/s (%sp/s avg); " + "app success: %llu %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%% " + "app hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_avg_str, exp->send_bandwidth_avg_str, + exp->recv_success_unique, exp->recv_rate_str, exp->recv_avg_str, + exp->app_recv_success_unique, exp->app_success_rate_str, + exp->app_success_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate, exp->app_hitrate); + } +} + +static void onscreen_generic(export_status_t *exp) { + if (!exp->complete) { + fprintf(stderr, + "%5s %0.0f%%%s; " + "send: %llu %sp/s %sb/s (%sp/s %sb/s avg); " + "recv: %llu %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, exp->send_rate_str, + exp->send_bandwidth_str, exp->send_rate_avg_str, + exp->send_bandwidth_avg_str, exp->recv_success_unique, + exp->recv_rate_str, exp->recv_avg_str, exp->pcap_drop_last_str, + exp->pcap_drop_avg_str, exp->hitrate); + } else { + fprintf(stderr, + "%5s %0.0f%%%s; " + "send: %llu done (%sp/s %sb/s avg); " + "recv: %llu %sp/s (%sp/s avg); " + "drops: %sp/s (%sp/s avg); " + "hitrate: %0.2f%%\n", + exp->time_past_str, exp->percent_complete, + exp->time_remaining_str, exp->total_sent, + exp->send_rate_avg_str, exp->send_bandwidth_avg_str, + exp->recv_success_unique, exp->recv_rate_str, exp->recv_avg_str, + exp->pcap_drop_last_str, exp->pcap_drop_avg_str, exp->hitrate); + } + fflush(stderr); +} + +static FILE *init_status_update_file(char *path) { + FILE *f = fopen(path, "w"); + if (!f) { + log_fatal("csv", "could not open status updates file (%s): %s", + xconf.status_updates_file, strerror(errno)); + } + log_debug("monitor", "status updates CSV will be saved to %s", + xconf.status_updates_file); + fprintf( + f, + "real-time,time-elapsed,time-remaining," + "percent-complete,hit-rate,active-send-threads," + "sent-total,sent-last-one-sec,sent-avg-per-sec," + "sent-bandwidth-last-one-sec,sent-bandwidth-avg-per-sec," + "recv-success-total,recv-success-last-one-sec,recv-success-avg-per-sec," + "recv-total,recv-total-last-one-sec,recv-total-avg-per-sec," + "pcap-drop-total,drop-last-one-sec,drop-avg-per-sec," + "sendto-fail-total,sendto-fail-last-one-sec,sendto-fail-avg-per-sec\n"); + fflush(f); + + return f; +} + +static void update_status_updates_file(export_status_t *exp, FILE *f) { + struct timeval now; + char timestamp[256]; + gettimeofday(&now, NULL); + time_t sec = now.tv_sec; + struct tm *ptm = localtime(&sec); + strftime(timestamp, 20, "%Y-%m-%d %H:%M:%S", ptm); + + fprintf(f, + "%s,%u,%u," + "%f,%f,%u," + "%llu,%.0f,%.0f," + "%.0f,%.0f," + "%llu,%.0f,%.0f," + "%llu,%.0f,%.0f," + "%llu,%.0f,%.0f," + "%llu,%.0f,%.0f\n", + timestamp, exp->time_past, exp->time_remaining, + exp->percent_complete, exp->hitrate, exp->send_threads, + exp->total_sent, exp->send_rate, exp->send_rate_avg, + exp->send_bandwidth, exp->send_bandwidth_avg, + exp->recv_success_unique, exp->recv_rate, exp->recv_avg, + exp->total_recv, exp->recv_total_rate, exp->recv_total_avg, + exp->pcap_drop_total, exp->pcap_drop_last, exp->pcap_drop_avg, + exp->fail_total, exp->fail_last, exp->fail_avg); + fflush(f); +} + +static inline void check_min_hitrate(export_status_t *exp) { + if (exp->seconds_under_min_hitrate >= MIN_HITRATE_TIME_WINDOW) { + log_fatal("monitor", + "hitrate below %.0f for %.0f seconds. aborting scan.", + xconf.min_hitrate, exp->seconds_under_min_hitrate); + } +} + +static inline void check_max_sendto_failures(export_status_t *exp) { + if (xconf.max_sendto_failures >= 0 && + exp->fail_total > (uint32_t) xconf.max_sendto_failures) { + log_fatal("monitor", "maximum number of sendto failures (%i) exceeded", + xconf.max_sendto_failures); + } +} + +void monitor_init(void) { + if (xconf.status_updates_file) { + status_fd = init_status_update_file(xconf.status_updates_file); + assert(status_fd); + } +} + +void monitor_run(iterator_t *it, pthread_mutex_t *lock) { + log_debug("monitor", "thread started"); + + int_status_t * internal_status = xmalloc(sizeof(int_status_t)); + export_status_t *export_status = xmalloc(sizeof(export_status_t)); + + while (!(xsend.complete && xrecv.complete)) { + update_pcap_stats(lock); + export_stats(internal_status, export_status, it); + log_drop_warnings(export_status); + check_min_hitrate(export_status); + check_max_sendto_failures(export_status); + if (!xconf.quiet) { + lock_file(stderr); + if (xconf.fsconf.app_success_index >= 0) { + onscreen_appsuccess(export_status); + } else { + onscreen_generic(export_status); + } + unlock_file(stderr); + } + if (status_fd) { + update_status_updates_file(export_status, status_fd); + } + sleep(UPDATE_INTERVAL); + } + if (!xconf.quiet) { + lock_file(stderr); + fflush(stderr); + unlock_file(stderr); + } + if (status_fd) { + fflush(status_fd); + fclose(status_fd); + } + + log_debug("monitor", "thread completed"); +} diff --git a/src/monitor.h b/src/monitor.h new file mode 100644 index 0000000..276f2fb --- /dev/null +++ b/src/monitor.h @@ -0,0 +1,23 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MONITOR_H +#define XMAP_MONITOR_H + +#include + +#include "iterator.h" + +void monitor_run(iterator_t *it, pthread_mutex_t *lock); + +void monitor_init(void); + +#endif // XMAP_MONITOR_H diff --git a/src/output_modules/module_csv.c b/src/output_modules/module_csv.c new file mode 100644 index 0000000..c5a9979 --- /dev/null +++ b/src/output_modules/module_csv.c @@ -0,0 +1,130 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "module_csv.h" + +#include +#include +#include +#include +#include +#include + +#include "../../lib/logger.h" +#include "output_modules.h" + +static FILE *file = NULL; + +int csv_init(struct state_conf *conf, char **fields, int fieldlens) { + assert(conf); + + if (conf->output_filename) { + if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal("csv", "could not open CSV output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + } else { + file = stdout; + log_info("csv", "no output file selected, will use stdout"); + } + + if (file && strcmp(conf->output_module_name, "default")) { + log_debug("csv", "more than one field, will add headers"); + for (int i = 0; i < fieldlens; i++) { + if (i) { + fprintf(file, ","); + } + fprintf(file, "%s", fields[i]); + } + fprintf(file, "\n"); + } + + check_and_log_file_error(file, "csv"); + + return EXIT_SUCCESS; +} + +int csv_close(__attribute__((unused)) struct state_conf *c, + __attribute__((unused)) struct state_send *s, + __attribute__((unused)) struct state_recv *r) { + if (file) { + fflush(file); + fclose(file); + } + + return EXIT_SUCCESS; +} + +static void hex_encode(FILE *f, unsigned char *readbuf, size_t len) { + for (size_t i = 0; i < len; i++) { + fprintf(f, "%02x", readbuf[i]); + } + check_and_log_file_error(f, "csv"); +} + +int csv_process(fieldset_t *fs) { + if (!file) { + return EXIT_SUCCESS; + } + + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (i) { + fprintf(file, ","); + } + + if (f->type == FS_STRING) { + if (strchr((char *) f->value.ptr, ',')) { + fprintf(file, "\"%s\"", (char *) f->value.ptr); + } else { + fprintf(file, "%s", (char *) f->value.ptr); + } + } else if (f->type == FS_UINT64) { + fprintf(file, "%" PRIu64, (uint64_t) f->value.num); + } else if (f->type == FS_BOOL) { + fprintf(file, "%" PRIi32, (int) f->value.num); + } else if (f->type == FS_BINARY) { + hex_encode(file, (unsigned char *) f->value.ptr, f->len); + } else if (f->type == FS_NULL) { + // do nothing + } else { + log_fatal("csv", "received unknown output type: %d (%s)", f->type, + f->name); + } + } + fprintf(file, "\n"); + fflush(file); + check_and_log_file_error(file, "csv"); + + return EXIT_SUCCESS; +} + +output_module_t module_csv_file = { + .name = "csv", + .filter_duplicates = 0, // framework should not filter out duplicates + .filter_unsuccessful = 0, // framework should not filter out unsuccessful + .init = &csv_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &csv_close, + .process_ip = &csv_process, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .helptext = "Outputs one or more output fields as a comma-delimited file. " + "By default, the \n" + "probe module does not filter out duplicates or limit to " + "successful fields, \n" + "but rather includes all received packets. Fields can be " + "controlled by \n" + "setting --output-fields. Filtering out failures and duplicate " + "packets can \n" + "be achieved by setting an --output-filter."}; diff --git a/src/output_modules/module_csv.h b/src/output_modules/module_csv.h new file mode 100644 index 0000000..830956a --- /dev/null +++ b/src/output_modules/module_csv.h @@ -0,0 +1,22 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_CSV_H +#define XMAP_MODULE_CSV_H + +#include "output_modules.h" + +#include "../fieldset.h" + +int csv_init(struct state_conf *conf, char **fields, int fieldlens); + +int csv_process(fieldset_t *fs); + +int csv_close(struct state_conf *c, struct state_send *s, struct state_recv *r); + +#endif // XMAP_MODULE_CSV_H diff --git a/src/output_modules/module_json.c b/src/output_modules/module_json.c new file mode 100644 index 0000000..e4aef68 --- /dev/null +++ b/src/output_modules/module_json.c @@ -0,0 +1,152 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "module_json.h" + +#include +#include +#include +#include +#include +#include + +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "output_modules.h" + +static FILE *file = NULL; + +int json_output_file_init(struct state_conf *conf, UNUSED char **fields, + UNUSED int fieldlens) { + assert(conf); + + if (!conf->output_filename) { + file = stdout; + } else if (!strcmp(conf->output_filename, "-")) { + file = stdout; + } else { + if (!(file = fopen(conf->output_filename, "w"))) { + log_fatal("output-json", "could not open JSON output file (%s): %s", + conf->output_filename, strerror(errno)); + } + } + check_and_log_file_error(file, "json"); + + return EXIT_SUCCESS; +} + +char *hex_encode(unsigned char *packet, int buflen) { + char *buf = xmalloc(2 * buflen + 1); + for (int i = 0; i < buflen; i++) { + snprintf(buf + (i * 2), 3, "%.2x", packet[i]); + } + buf[buflen * 2] = 0; + + return buf; +} + +json_object *fs_to_jsonobj(fieldset_t *fs); + +json_object *repeated_to_jsonobj(fieldset_t *fs); + +json_object *field_to_jsonobj(field_t *f) { + if (f->type == FS_STRING) { + return json_object_new_string((char *) f->value.ptr); + } else if (f->type == FS_UINT64) { + return json_object_new_int64(f->value.num); + } else if (f->type == FS_BOOL) { + return json_object_new_boolean(f->value.num); + } else if (f->type == FS_BINARY) { + char * encoded = hex_encode(f->value.ptr, f->len); + json_object *t = json_object_new_string(encoded); + free(encoded); + return t; + } else if (f->type == FS_NULL) { + return NULL; + } else if (f->type == FS_FIELDSET) { + return fs_to_jsonobj((fieldset_t *) f->value.ptr); + } else if (f->type == FS_REPEATED) { + return repeated_to_jsonobj((fieldset_t *) f->value.ptr); + } else { + log_fatal("json", "received unknown output type: %i", f->type); + } +} + +json_object *repeated_to_jsonobj(fieldset_t *fs) { + json_object *obj = json_object_new_array(); + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + json_object_array_add(obj, field_to_jsonobj(f)); + } + + return obj; +} + +json_object *fs_to_jsonobj(fieldset_t *fs) { + json_object *obj = json_object_new_object(); + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + json_object_object_add(obj, f->name, field_to_jsonobj(f)); + } + + return obj; +} + +int json_output_to_file(fieldset_t *fs) { + if (!file) { + return EXIT_SUCCESS; + } + + json_object *record = fs_to_jsonobj(fs); + fprintf(file, "%s\n", json_object_to_json_string(record)); + fflush(file); + check_and_log_file_error(file, "json"); + json_object_put(record); + + return EXIT_SUCCESS; +} + +int json_output_file_close(UNUSED struct state_conf *c, + UNUSED struct state_send *s, + UNUSED struct state_recv *r) { + if (file) { + fflush(file); + fclose(file); + } + + return EXIT_SUCCESS; +} + +int print_json_fieldset(fieldset_t *fs) { + json_object *record = fs_to_jsonobj(fs); + fprintf(stdout, "%s\n", json_object_to_json_string(record)); + json_object_put(record); + + return EXIT_SUCCESS; +} + +output_module_t module_json_file = { + .name = "json", + .init = &json_output_file_init, + .filter_duplicates = 0, // framework should not filter out duplicates + .filter_unsuccessful = 0, // framework should not filter out unsuccessful + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &json_output_file_close, + .process_ip = &json_output_to_file, + .supports_dynamic_output = DYNAMIC_SUPPORT, + .helptext = "Outputs one or more output fields as a json valid file. By " + "default, the \n" + "probe module does not filter out duplicates or limit to " + "successful fields, \n" + "but rather includes all received packets. Fields can be " + "controlled by \n" + "setting --output-fields. Filtering out failures and duplicate " + "packets can \n" + "be achieved by setting an --output-filter."}; diff --git a/src/output_modules/module_json.h b/src/output_modules/module_json.h new file mode 100644 index 0000000..cfbbdf3 --- /dev/null +++ b/src/output_modules/module_json.h @@ -0,0 +1,16 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_JSON_H +#define XMAP_MODULE_JSON_H + +#include "../fieldset.h" + +int print_json_fieldset(fieldset_t *fs); + +#endif // XMAP_MODULE_JSON_H diff --git a/src/output_modules/module_redis_csv.c b/src/output_modules/module_redis_csv.c new file mode 100644 index 0000000..338ead7 --- /dev/null +++ b/src/output_modules/module_redis_csv.c @@ -0,0 +1,222 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "module_redis_csv.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../../lib/logger.h" +#include "../../lib/redis.h" +#include "../../lib/xalloc.h" + +#define UNUSED __attribute__((unused)) + +#define BUFFER_SIZE 1000 + +static char ** buffer; +static int buffer_fill = 0; +static char * queue_name = NULL; +static redisContext *rctx = NULL; + +int rediscsvmodule_init(struct state_conf *conf, UNUSED char **fields, + UNUSED int fieldlens) { + // This function leaks memory but not much + buffer = xcalloc(BUFFER_SIZE, sizeof(char *)); + buffer_fill = 0; + redisconf_t c; + redisconf_t *rconf = &c; + char * connect_string = NULL; + if (conf->output_args) { + log_debug("redis-csv", "output args %s", conf->output_args); + connect_string = conf->output_args; + } else { + connect_string = strdup("local:///tmp/redis.sock/xmap"); + } + if (redis_parse_connstr(connect_string, rconf) != PMAP_REDIS_SUCCESS) { + log_error("redis-csv", "error parsing connect string (%s)", + rconf->error); + return EXIT_FAILURE; + } + if (rconf->type == T_TCP) { + log_info("redis-csv", + "{type: TCP, server: %s, " + "port: %u, list: %s}", + rconf->server, rconf->port, rconf->list_name); + } else { + log_info("redis-csv", + "{type: LOCAL, path: %s, " + "list: %s}", + rconf->path, rconf->list_name); + } + + if (rconf && rconf->list_name) { + queue_name = rconf->list_name; + } else { + queue_name = strdup("xmap"); + } + + // generate field names CSV list to be logged. + char *fieldstring = xcalloc(1000, fieldlens); + memset(fieldstring, 0, sizeof(fields)); + for (int i = 0; i < fieldlens; i++) { + if (i) { + strcat(fieldstring, ", "); + } + strcat(fieldstring, fields[i]); + } + log_info("redis-csv", "the following fields will be output to redis: %s.", + fieldstring); + free(fields); + + rctx = redis_connect_from_conf(rconf); + if (!rctx) { + log_error("redis-csv", "could not connect to redis"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +static int rediscsvmodule_flush(void) { + if (redis_rpush_strings(rctx, (char *) queue_name, buffer, buffer_fill)) { + return EXIT_FAILURE; + } + for (int i = 0; i < buffer_fill; i++) { + free(buffer[i]); + } + buffer_fill = 0; + return EXIT_SUCCESS; +} + +#define INT_STR_LEN 20 // len(9223372036854775807) == 19 + +static size_t guess_csv_string_length(fieldset_t *fs) { + size_t len = 0; + for (int i = 0; i < fs->len; i++) { + field_t *f = &(fs->fields[i]); + if (f->type == FS_STRING) { + len += strlen(f->value.ptr); + len += 2; // potential quotes + } else if (f->type == FS_UINT64) { + len += INT_STR_LEN; + } else if (f->type == FS_BOOL) { + len += INT_STR_LEN; // 0 or 1 PRIi32 is used to print ... + } else if (f->type == FS_BINARY) { + len += 2 * f->len; + } else if (f->type == FS_NULL) { + // do nothing + } else { + log_fatal("csv", "received unknown output type " + "(not str, binary, null, or uint64_t)"); + } + } + // estimated length + number of commas + return len + (size_t) len + 256; +} + +static void hex_encode_str(char *f, unsigned char *readbuf, size_t len) { + char *temp = f; + for (size_t i = 0; i < len; i++) { + sprintf(temp, "%02x", readbuf[i]); + temp += (size_t) 2 * sizeof(char); + } +} + +void make_csv_string(fieldset_t *fs, char *out, size_t len) { + memset(out, 0, len); + for (int i = 0; i < fs->len; i++) { + char * temp = out + (size_t) strlen(out); + field_t *f = &(fs->fields[i]); + char * dataloc = temp; + if (i) { // only add comma if not first element + sprintf(temp, ","); + dataloc += (size_t) 1; + } + if (f->type == FS_STRING) { + if (strlen(dataloc) + strlen((char *) f->value.ptr) >= len) { + log_fatal("redis-csv", "out of memory---will overflow"); + } + if (strchr((char *) f->value.ptr, ',')) { + sprintf(dataloc, "\"%s\"", (char *) f->value.ptr); + } else { + sprintf(dataloc, "%s", (char *) f->value.ptr); + } + } else if (f->type == FS_UINT64) { + if (strlen(dataloc) + INT_STR_LEN >= len) { + log_fatal("redis-csv", "out of memory---will overflow"); + } + sprintf(dataloc, "%" PRIu64, (uint64_t) f->value.num); + } else if (f->type == FS_BOOL) { + if (strlen(dataloc) + INT_STR_LEN >= len) { + log_fatal("redis-csv", "out of memory---will overflow"); + } + sprintf(dataloc, "%" PRIi32, (int) f->value.num); + } else if (f->type == FS_BINARY) { + if (strlen(dataloc) + 2 * f->len >= len) { + log_fatal("redis-csv", "out of memory---will overflow"); + } + hex_encode_str(dataloc, (unsigned char *) f->value.ptr, f->len); + } else if (f->type == FS_NULL) { + // do nothing + } else { + log_fatal("redis-csv", "received unknown output type"); + } + } +} + +int rediscsvmodule_process(fieldset_t *fs) { + size_t reqd_space = guess_csv_string_length(fs); + char * x = xmalloc(reqd_space); + make_csv_string(fs, x, reqd_space); + buffer[buffer_fill] = x; + // if full, flush all to redis + if (++buffer_fill == BUFFER_SIZE) { + if (rediscsvmodule_flush()) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +int rediscsvmodule_close(UNUSED struct state_conf *c, + UNUSED struct state_send *s, + UNUSED struct state_recv *r) { + if (rediscsvmodule_flush()) { + return EXIT_FAILURE; + } + if (redis_close(rctx)) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +output_module_t module_redis_csv = { + .name = "redis-csv", + .init = &rediscsvmodule_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .close = &rediscsvmodule_close, + .process_ip = &rediscsvmodule_process, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .helptext = "Outputs one or more output fields in csv, and then flushes " + "out to redis. \n" + "By default, the probe module does not filter out duplicates " + "or limit to successful fields, \n" + "but rather includes all received packets. Fields can be " + "controlled by \n" + "setting --output-fileds. Filtering out failures and duplicate " + "packets can \n" + "be achieved by setting an --output-filter."}; diff --git a/src/output_modules/module_redis_csv.h b/src/output_modules/module_redis_csv.h new file mode 100644 index 0000000..6fd9c1c --- /dev/null +++ b/src/output_modules/module_redis_csv.h @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_REDIS_CSV_H +#define XMAP_MODULE_REDIS_CSV_H + +#include "output_modules.h" + +int redisstrmodule_init(struct state_conf *conf, char **fields, int fieldlens); + +int redisstrmodule_process(fieldset_t *fs); + +int redisstrmodule_close(struct state_conf *c, struct state_send *s, + struct state_recv *r); + +#endif //XMAP_MODULE_REDIS_CSV_H diff --git a/src/output_modules/module_redis_packed.c b/src/output_modules/module_redis_packed.c new file mode 100644 index 0000000..a03d7f7 --- /dev/null +++ b/src/output_modules/module_redis_packed.c @@ -0,0 +1,130 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "module_redis_packed.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "../../lib/logger.h" +#include "../../lib/redis.h" +#include "../../lib/xalloc.h" + +#include "output_modules.h" + +#define UNUSED __attribute__((unused)) + +#define BUFFER_SIZE 500 + +static uint32_t * buffer; +static int buffer_fill = 0; +static char * queue_name = NULL; +static int field_index = -1; +static redisContext *rctx = NULL; + +int redismodule_init(struct state_conf *conf, char **fields, int fieldlens) { + buffer = xcalloc(BUFFER_SIZE, sizeof(uint32_t)); + buffer_fill = 0; + for (int i = 0; i < fieldlens; i++) { + if (!strcmp(fields[i], "saddr_raw")) { + field_index = i; + break; + } + } + if (field_index < 0) { + log_fatal("redis-module", "saddr_raw not included in output-fields"); + } + + redisconf_t rconf; + memset(&rconf, 0, sizeof(redisconf_t)); + char *connect_string = NULL; + if (conf->output_args) { + connect_string = conf->output_args; + } else { + connect_string = strdup("local:///tmp/redis.sock/xmap_output"); + } + + if (redis_parse_connstr(connect_string, &rconf) != PMAP_REDIS_SUCCESS) { + log_error("redis-module", "configuration error: %s", rconf.error); + return EXIT_FAILURE; + } + + if (rconf.list_name) { + queue_name = rconf.list_name; + } else { + queue_name = strdup("xmap_output"); + } + + if (rconf.type == T_TCP) { + log_info("redis-module", + "{type: TCP, server: %s, " + "port: %u, list: %s}", + rconf.server, rconf.port, rconf.list_name); + } else { + log_info("redis-module", + "{type: LOCAL, path: %s, " + "list: %s}", + rconf.path, queue_name); + } + + rctx = redis_connect_from_conf(&rconf); + if (!rctx) { + log_error("redis-module", "could not connect to redis"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +static int redismodule_flush(void) { + if (redis_rpush(rctx, (char *) queue_name, buffer, buffer_fill, + sizeof(uint32_t))) { + return EXIT_FAILURE; + } + buffer_fill = 0; + return EXIT_SUCCESS; +} + +int redismodule_process(fieldset_t *fs) { + field_t *f = &(fs->fields[field_index]); + buffer[buffer_fill] = (uint32_t) f->value.num; + if (++buffer_fill == BUFFER_SIZE) { + if (redismodule_flush()) { + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +int redismodule_close(UNUSED struct state_conf *c, UNUSED struct state_send *s, + UNUSED struct state_recv *r) { + if (redismodule_flush()) { + return EXIT_FAILURE; + } + if (redis_close(rctx)) { + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +output_module_t module_redis = {.name = "redis-packed", + .init = &redismodule_init, + .start = NULL, + .update = NULL, + .update_interval = 0, + .supports_dynamic_output = NO_DYNAMIC_SUPPORT, + .close = &redismodule_close, + .process_ip = &redismodule_process, + .helptext = + "Flushes to redis the ipv46 address as " + "packed binary integer in network order\n"}; diff --git a/src/output_modules/module_redis_packed.h b/src/output_modules/module_redis_packed.h new file mode 100644 index 0000000..6c4bb1d --- /dev/null +++ b/src/output_modules/module_redis_packed.h @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_REDIS_PACKED_H +#define XMAP_MODULE_REDIS_PACKED_H + +#include "output_modules.h" + +int redismodule_init(struct state_conf *conf, char **fields, int fieldlens); + +int redismodule_process(fieldset_t *fs); + +int redismodule_close(struct state_conf *c, struct state_send *s, + struct state_recv *r); + +#endif //XMAP_MODULE_REDIS_PACKED_H diff --git a/src/output_modules/output_modules.c b/src/output_modules/output_modules.c new file mode 100644 index 0000000..413b019 --- /dev/null +++ b/src/output_modules/output_modules.c @@ -0,0 +1,62 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "output_modules.h" + +#include +#include + +extern output_module_t module_csv_file; + +#ifdef JSON +extern output_module_t module_json_file; +#endif + +#ifdef REDIS +extern output_module_t module_redis; +extern output_module_t module_redis_csv; +extern output_module_t module_csv_redis; +#endif + +#ifdef MONGODB +extern output_module_t module_mongodb; +#endif + +output_module_t *output_modules[] = { + &module_csv_file, +#ifdef JSON + &module_json_file, +#endif +#ifdef REDIS + &module_redis, &module_redis_csv, +#endif +#ifdef MONGODB + &module_mongodb, +#endif + // ADD YOUR MODULE HERE +}; + +output_module_t *get_output_module_by_name(const char *name) { + int num_modules = + (int) (sizeof(output_modules) / sizeof(output_modules[0])); + for (int i = 0; i < num_modules; i++) { + if (!strcmp(output_modules[i]->name, name)) { + return output_modules[i]; + } + } + + return NULL; +} + +void print_output_modules(void) { + int num_modules = + (int) (sizeof(output_modules) / sizeof(output_modules[0])); + for (int i = 0; i < num_modules; i++) { + printf("%s\n", output_modules[i]->name); + } +} diff --git a/src/output_modules/output_modules.h b/src/output_modules/output_modules.h new file mode 100644 index 0000000..1943768 --- /dev/null +++ b/src/output_modules/output_modules.h @@ -0,0 +1,47 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_OUTPUT_MODULES_H +#define XMAP_OUTPUT_MODULES_H + +#include "../fieldset.h" +#include "../state.h" + +#define NO_DYNAMIC_SUPPORT 0 +#define DYNAMIC_SUPPORT 1 + +// called at scanner initialization +typedef int (*output_init_cb)(struct state_conf *, char **fields, + int fieldslen); + +// called on packet receipt +typedef int (*output_packet_cb)(fieldset_t *fs); + +// called periodically during the scan +typedef int (*output_update_cb)(struct state_conf *, struct state_send *, + struct state_recv *); + +typedef struct output_module { + const char * name; + int supports_dynamic_output; + int filter_duplicates; + int filter_unsuccessful; + unsigned update_interval; + output_init_cb init; + output_update_cb start; + output_update_cb update; + output_update_cb close; + output_packet_cb process_ip; + const char * helptext; +} output_module_t; + +output_module_t *get_output_module_by_name(const char *); + +void print_output_modules(void); + +#endif // XMAP_OUTPUT_MODULES_H diff --git a/src/parser.y b/src/parser.y new file mode 100644 index 0000000..86b7fa5 --- /dev/null +++ b/src/parser.y @@ -0,0 +1,140 @@ +%{ +#include +#include +#include "expression.h" +#include "lexer.h" +#include "filter.h" + +void yyerror(const char *str) { + fprintf(stderr,"Parse error: %s\n",str); +} + +int yywrap() { + return 1; +} + +extern node_t *zfilter; + +%} + +%union { + int int_literal; + char *string_literal; + struct node_st *expr; +} + +%token '(' ')' T_AND T_OR +%token T_NUMBER +%token T_FIELD +%token T_NOT_EQ T_GT_EQ '>' '<' '=' T_LT_EQ + +%left T_OR +%left T_AND + +%type filter +%type number_filter +%type string_filter +%type filter_expr + + +%% + +expression: filter_expr + { + zfilter = $1; + } + + +filter_expr: + filter_expr T_OR filter_expr + { + $$ = make_op_node(OR); + $$->left_child = $1; + $$->right_child = $3; + } + | filter_expr T_AND filter_expr + { + $$ = make_op_node(AND); + $$->left_child = $1; + $$->right_child = $3; + } + | '(' filter_expr ')' + { + $$ = $2; + } + | filter + { + $$ = $1; + } + ; + +filter: number_filter + { + $$ = $1; + } + | string_filter + { + $$ = $1; + } + ; + +number_filter: T_FIELD '=' T_NUMBER + { + $$ = make_op_node(EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD '>' T_NUMBER + { + $$ = make_op_node(GT); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD '<' T_NUMBER + { + $$ = make_op_node(LT); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_NOT_EQ T_NUMBER + { + $$ = make_op_node(NEQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_GT_EQ T_NUMBER + { + $$ = make_op_node(GT_EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + | + T_FIELD T_LT_EQ T_NUMBER + { + $$ = make_op_node(LT_EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_int_node($3); + } + ; + +string_filter: + T_FIELD '=' T_FIELD + { + $$ = make_op_node(EQ); + $$->left_child = make_field_node($1); + $$->right_child = make_string_node($3); + } + | + T_FIELD T_NOT_EQ T_FIELD + { + $$ = make_op_node(NEQ); + $$->left_child = make_field_node($1); + $$->right_child = make_string_node($3); + } + ; + +%% diff --git a/src/probe_modules/module_icmp6.h b/src/probe_modules/module_icmp6.h new file mode 100644 index 0000000..59e4b3e --- /dev/null +++ b/src/probe_modules/module_icmp6.h @@ -0,0 +1,29 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_ICMP6_H +#define XMAP_MODULE_ICMP6_H + +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../../lib/types.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" +#include "../validate.h" +#include "packet.h" +#include "packet_icmp6.h" +#include "probe_modules.h" + +#define ICMP6_MINLEN 8 +#define ICMP6_MAX_PAYLOAD_LEN 1500 - 40 - 8 // 1500 - IPv6_h - ICMPv6_h + +#endif // XMAP_MODULE_ICMP6_H diff --git a/src/probe_modules/module_icmp6_echo.c b/src/probe_modules/module_icmp6_echo.c new file mode 100644 index 0000000..e6808c8 --- /dev/null +++ b/src/probe_modules/module_icmp6_echo.c @@ -0,0 +1,382 @@ +/* + * ZMapv6 Copyright 2016 Chair of Network Architectures and Services + * Technical University of Munich + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMPv6 echo request (ping) scans + +#include "module_icmp6.h" + +probe_module_t module_icmp6_echo; + +static const char *icmp6_usage_error = + "unknown ICMPv6 probe specification (expected file:/path or text:STRING or " + "hex:01020304 or icmpv6-type-code-str)"; + +static size_t icmp6_payload_len = 0; +static const size_t icmp6_payload_default_len = 8; +static char * icmp6_payload = NULL; + +static int icmp6_echo_global_init(struct state_conf *conf) { + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) { + icmp6_payload = xmalloc(icmp6_payload_default_len); + icmp6_payload_len = icmp6_payload_default_len; + return EXIT_SUCCESS; + } + + if (strcmp(conf->probe_args, "icmp-type-code-str") == 0) { + print_icmp6_type_code_str(); + exit(EXIT_SUCCESS); + } + + char *c = strchr(conf->probe_args, ':'); + if (!c) { + log_error("icmp6", icmp6_usage_error); + return EXIT_FAILURE; + } + ++c; + + if (strncmp(conf->probe_args, "text", 4) == 0) { + icmp6_payload = strdup(c); + icmp6_payload_len = strlen(icmp6_payload); + } else if (strncmp(conf->probe_args, "file", 4) == 0) { + FILE *inp = fopen(c, "rb"); + if (!inp) { + log_error("icmp6", "could not open ICMPv6 data file '%s'", c); + free(icmp6_payload); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_END)) { + log_error("icmp6", "unable to get size of ICMPv6 data file '%s'", + c); + free(icmp6_payload); + return EXIT_FAILURE; + } + size_t input_size = ftell(inp); + if (input_size > ICMP6_MAX_PAYLOAD_LEN) { + log_error("icmp6", + "input file larger than %d bytes and will not fit on the " + "wire (%llu bytes provided)", + ICMP6_MAX_PAYLOAD_LEN, input_size); + free(icmp6_payload); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_SET)) { + log_error("icmp6", "unable to read ICMPv6 data file '%s'", c); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload = xmalloc(ICMP6_MAX_PAYLOAD_LEN); + icmp6_payload_len = fread(icmp6_payload, 1, ICMP6_MAX_PAYLOAD_LEN, inp); + fclose(inp); + } else if (strncmp(conf->probe_args, "hex", 3) == 0) { + if (strlen(c) % 2 != 0) { + log_error("icmp6", + "invalid hex input (length must be a multiple of 2)"); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload_len = strlen(c) / 2; + icmp6_payload = xmalloc(icmp6_payload_len); + + unsigned int n; + for (size_t i = 0; i < icmp6_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_error("icmp6", "non-hex character: '%c'", c[i * 2]); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload[i] = (char) (n & 0xff); + } + } else { + log_error("icmp6", icmp6_usage_error); + free(icmp6_payload); + return EXIT_FAILURE; + } + + if (icmp6_payload_len > ICMP6_MAX_PAYLOAD_LEN) { + log_error("icmp6", + "reducing ICMPv6 payload must be at most %d bytes to fit on " + "the wire (%d were provided)\n", + ICMP6_MAX_PAYLOAD_LEN, icmp6_payload_len); + free(icmp6_payload); + return EXIT_FAILURE; + } + + module_icmp6_echo.packet_length = sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + ICMP6_MINLEN + + icmp6_payload_len; + assert(module_icmp6_echo.packet_length <= + 1500 + sizeof(struct ether_header)); + module_icmp6_echo.pcap_snaplen = + sizeof(struct ether_header) + + 2 * (sizeof(struct ip6_hdr) + ICMP6_MINLEN) + icmp6_payload_len; + + return EXIT_SUCCESS; +} + +static int icmp6_echo_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv) { + if (icmp6_payload) { + free(icmp6_payload); + icmp6_payload = NULL; + } + + return EXIT_SUCCESS; +} + +static int icmp6_echo_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth6_header(eth_header, src, gw); + + struct ip6_hdr *ip6_header = (struct ip6_hdr *) (ð_header[1]); + uint16_t payload_len = sizeof(struct icmp6_hdr) + icmp6_payload_len; + make_ip6_header(ip6_header, IPPROTO_ICMPV6, payload_len); + + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + make_icmp6_header(icmp6_header); + + char *payload = (char *) icmp6_header + ICMP6_MINLEN; + memcpy(payload, icmp6_payload, icmp6_payload_len); + + return EXIT_SUCCESS; +} + +static int icmp6_echo_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + UNUSED port_h_t dst_port, uint8_t ttl, + UNUSED int probe_num, UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) (ð_header[1]); + struct icmp6_hdr * icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, 0, validation); + + uint16_t icmp6_idnum = icmp_get_idnum(validation); + uint16_t icmp6_seqnum = icmp_get_seqnum(validation); + + uint8_t *ip6_src = (uint8_t *) &(ip6_header->ip6_src); + uint8_t *ip6_dst = (uint8_t *) &(ip6_header->ip6_dst); + for (int i = 0; i < 16; i++) { + ip6_src[i] = src_ip[i]; + ip6_dst[i] = dst_ip[i]; + } + ip6_header->ip6_hlim = ttl; + + icmp6_header->icmp6_id = icmp6_idnum; + icmp6_header->icmp6_seq = icmp6_seqnum; + + icmp6_header->icmp6_cksum = 0; + icmp6_header->icmp6_cksum = + (uint16_t) icmp6_checksum((struct in6_addr *) &(ip6_header->ip6_src), + (struct in6_addr *) &(ip6_header->ip6_dst), + icmp6_header, icmp6_payload_len); + + return EXIT_SUCCESS; +} + +static void icmp6_echo_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) ð_header[1]; + struct icmp6_hdr * icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + + fprintf_eth_header(fp, eth_header); + fprintf_ip6_header(fp, ip6_header); + fprintf(fp, + "ICMPv6\n" + "\tType(1B)\t\t: %u\n" + "\tCode(1B)\t\t: %u\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tIdentifier(2B)\t\t: n:%d (0x%04x) h:%d (0x%04x)\n" + "\tSequence number(2B)\t: n:%d (0x%04x) h:%d (0x%04x)\n", + icmp6_header->icmp6_type, icmp6_header->icmp6_code, + icmp6_header->icmp6_cksum, icmp6_header->icmp6_id, + icmp6_header->icmp6_id, ntohs(icmp6_header->icmp6_id), + ntohs(icmp6_header->icmp6_id), icmp6_header->icmp6_seq, + icmp6_header->icmp6_seq, ntohs(icmp6_header->icmp6_seq), + ntohs(icmp6_header->icmp6_seq)); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int icmp6_echo_validate_packet(const struct ip *ip_hdr, uint32_t len, + int *is_repeat) { + struct ip6_hdr *ip6_header = (struct ip6_hdr *) ip_hdr; + + if (ip6_header->ip6_nxt != IPPROTO_ICMPV6) { + return 0; + } + + // IPv6 header is fixed length at 40 bytes + ICMPv6 header + if ((sizeof(struct ip6_hdr) + sizeof(struct icmp6_hdr) + + icmp6_payload_len) > len) { + // buffer not large enough to contain expected icmp header + return 0; + } + + // ICMPv6 header + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + uint16_t icmp6_idnum = icmp6_header->icmp6_id; + uint16_t icmp6_seqnum = icmp6_header->icmp6_seq; + + // ICMPv6 validation is tricky: for some packet types, we must look inside + // the payload + uint8_t validation[VALIDATE_BYTES]; + if (icmp6_header->icmp6_type == ICMP6_TIME_EXCEEDED || + icmp6_header->icmp6_type == ICMP6_DST_UNREACH || + icmp6_header->icmp6_type == ICMP6_PACKET_TOO_BIG || + icmp6_header->icmp6_type == ICMP6_PARAM_PROB) { + + // IPv6 + ICMP6 headers + inner headers + payload (validation) + if (2 * sizeof(struct ip6_hdr) + 2 * sizeof(struct icmp6_hdr) + + icmp6_payload_len > + len) { + return 0; + } + + // Use inner headers for validation + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + struct icmp6_hdr *icmp6_inner_header = + (struct icmp6_hdr *) &ip6_inner_header[1]; + + // Regenerate validation and icmpv6 id based off inner payload + icmp6_idnum = icmp6_inner_header->icmp6_id; + icmp6_seqnum = icmp6_inner_header->icmp6_seq; + + // Send original src and dst IP as data in ICMPv6 payload and regenerate + // the validation here + validate_gen((const uint8_t *) &(ip6_inner_header->ip6_src), + (const uint8_t *) &(ip6_inner_header->ip6_dst), 0, + validation); + } else if (icmp6_header->icmp6_type == ICMP6_ECHO_REPLY) { + validate_gen((const uint8_t *) &(ip6_header->ip6_dst), + (const uint8_t *) &(ip6_header->ip6_src), 0, validation); + } + + // validate icmp id and seqnum + if (icmp6_idnum != icmp_get_idnum(validation)) { + return 0; + } + if (icmp6_seqnum != icmp_get_seqnum(validation)) { + return 0; + } + + // whether repeat: reply ip + char ip_port_str[xconf.ipv46_bytes]; + memcpy(ip_port_str, (char *) &(ip6_header->ip6_src), xconf.ipv46_bytes); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes); + *is_repeat = 0; + } + + return 1; +} + +static void icmp6_echo_process_packet(const u_char *packet, uint32_t len, + fieldset_t * fs, + UNUSED struct timespec ts) { + struct ip6_hdr *ip6_header = + (struct ip6_hdr *) &packet[sizeof(struct ether_header)]; + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + uint32_t hdrlen = sizeof(struct ether_header) + sizeof(struct ip6_hdr) + + 4; // after checksum + + if (icmp6_header->icmp6_type == ICMP6_ECHO_REPLY) { + fs_add_uint64(fs, "success", 1); + fs_add_string(fs, "clas", (char *) "echoreply", 0); + fs_add_string(fs, "desc", (char *) "no code", 0); + } else { + // Use inner IPv6 header values for unsuccessful ICMPv6 replies + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + fs_modify_string( + fs, "saddr", + make_ipv6_str((struct in6_addr *) &(ip6_inner_header->ip6_dst)), 1); + + fs_add_uint64(fs, "success", 0); + fs_add_string(fs, "clas", + (char *) get_icmp6_type_str(icmp6_header->icmp6_type), 0); + fs_add_string(fs, "desc", + (char *) get_icmp6_type_code_str( + icmp6_header->icmp6_type, icmp6_header->icmp6_code), + 0); + } + + fs_add_uint64(fs, "type", icmp6_header->icmp6_type); + fs_add_uint64(fs, "code", icmp6_header->icmp6_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp6_header->icmp6_id)); + fs_add_uint64(fs, "seq", ntohs(icmp6_header->icmp6_seq)); + fs_add_string(fs, "outersaddr", + make_ipv6_str((struct in6_addr *) &(ip6_header->ip6_src)), 1); + + int datalen = len - hdrlen; + + if (datalen > 0) { + const uint8_t *data = (uint8_t *) &packet[hdrlen]; + fs_add_binary(fs, "data", (size_t) datalen, (void *) data, 0); + } else { + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "success", + .type = "int", + .desc = "did probe module classify response as success"}, + {.name = "clas", + .type = "string", + .desc = "packet classification(type str):\n" + "\t\t\t`e.g., echoreply', `other'\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "desc", + .type = "string", + .desc = "ICMPv6 message detail(code str):\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "type", .type = "int", .desc = "ICMPv6 message type"}, + {.name = "code", .type = "int", .desc = "ICMPv6 message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "ICMPv6 id number"}, + {.name = "seq", .type = "int", .desc = "ICMPv6 sequence number"}, + {.name = "outersaddr", + .type = "string", + .desc = "outer src address of ICMPv6 reply packet"}, + {.name = "data", .type = "binary", .desc = "ICMPv6 payload"}, +}; + +probe_module_t module_icmp6_echo = { + .ipv46_flag = 6, + .name = "icmp_echo", + .packet_length = 14 + 40 + 8 + 8, + .pcap_filter = "icmp6 && (ip6[40] == 129 || ip6[40] == 3 || ip6[40] == 1 " + "|| ip6[40] == 2 || ip6[40] == 4)", + .pcap_snaplen = 14 + 2 * (40 + 8) + 8, + .port_args = 0, + .global_init = &icmp6_echo_global_init, + .close = &icmp6_echo_global_cleanup, + .thread_init = &icmp6_echo_thread_init, + .make_packet = &icmp6_echo_make_packet, + .print_packet = &icmp6_echo_print_packet, + .process_packet = &icmp6_echo_process_packet, + .validate_packet = &icmp6_echo_validate_packet, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = "Probe module that sends ICMPv6 echo requests to hosts.\n" + "Payload of ICMPv6 packets will consist of 8 bytes zero unless " + "you customize it with:\n" + " --probe-args=file:/path_to_payload_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164\n" + " --probe-args=icmp-type-code-str", +}; diff --git a/src/probe_modules/module_icmp6_echo_gw.c b/src/probe_modules/module_icmp6_echo_gw.c new file mode 100644 index 0000000..55a124c --- /dev/null +++ b/src/probe_modules/module_icmp6_echo_gw.c @@ -0,0 +1,384 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMPv6 echo request (ping) scans for gateway +// discovery + +#include "module_icmp6.h" + +probe_module_t module_icmp6_echo_gw; + +static const char *icmp6_usage_error = + "unknown ICMPv6 probe specification (expected file:/path or text:STRING or " + "hex:01020304 or icmpv6-type-code-str)"; + +static size_t icmp6_payload_len = 0; +static const size_t icmp6_payload_default_len = 8; +static char * icmp6_payload = NULL; + +static int icmp6_echo_gw_global_init(struct state_conf *conf) { + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) { + icmp6_payload = xmalloc(icmp6_payload_default_len); + icmp6_payload_len = icmp6_payload_default_len; + return EXIT_SUCCESS; + } + + if (strcmp(conf->probe_args, "icmp-type-code-str") == 0) { + print_icmp6_type_code_str(); + exit(EXIT_SUCCESS); + } + + char *c = strchr(conf->probe_args, ':'); + if (!c) { + log_error("icmp6", icmp6_usage_error); + return EXIT_FAILURE; + } + ++c; + + if (strncmp(conf->probe_args, "text", 4) == 0) { + icmp6_payload = strdup(c); + icmp6_payload_len = strlen(icmp6_payload); + } else if (strncmp(conf->probe_args, "file", 4) == 0) { + FILE *inp = fopen(c, "rb"); + if (!inp) { + log_error("icmp6", "could not open ICMPv6 data file '%s'", c); + free(icmp6_payload); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_END)) { + log_error("icmp6", "unable to get size of ICMPv6 data file '%s'", + c); + free(icmp6_payload); + return EXIT_FAILURE; + } + size_t input_size = ftell(inp); + if (input_size > ICMP6_MAX_PAYLOAD_LEN) { + log_error( + "icmp6", + "input file larger than %d bytes and will not fit on the wire " + "(%llu bytes provided)", + ICMP6_MAX_PAYLOAD_LEN, input_size); + free(icmp6_payload); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_SET)) { + log_error("icmp6", "unable to read ICMPv6 data file '%s'", c); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload = xmalloc(ICMP6_MAX_PAYLOAD_LEN); + icmp6_payload_len = fread(icmp6_payload, 1, ICMP6_MAX_PAYLOAD_LEN, inp); + fclose(inp); + } else if (strncmp(conf->probe_args, "hex", 3) == 0) { + if (strlen(c) % 2 != 0) { + log_error("icmp6", + "invalid hex input (length must be a multiple of 2)"); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload_len = strlen(c) / 2; + icmp6_payload = xmalloc(icmp6_payload_len); + + unsigned int n; + for (size_t i = 0; i < icmp6_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_error("icmp6", "non-hex character: '%c'", c[i * 2]); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload[i] = (char) (n & 0xff); + } + } else { + log_error("icmp6", icmp6_usage_error); + free(icmp6_payload); + return EXIT_FAILURE; + } + + if (icmp6_payload_len > ICMP6_MAX_PAYLOAD_LEN) { + log_error("icmp6", + "reducing ICMPv6 payload must be at most %d bytes to fit on " + "the wire (%d were provided)\n", + ICMP6_MAX_PAYLOAD_LEN, icmp6_payload_len); + free(icmp6_payload); + return EXIT_FAILURE; + } + + module_icmp6_echo_gw.packet_length = sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + ICMP6_MINLEN + + icmp6_payload_len; + assert(module_icmp6_echo_gw.packet_length <= + 1500 + sizeof(struct ether_header)); + module_icmp6_echo_gw.pcap_snaplen = + sizeof(struct ether_header) + + 2 * (sizeof(struct ip6_hdr) + ICMP6_MINLEN) + icmp6_payload_len; + + return EXIT_SUCCESS; +} + +static int icmp6_echo_gw_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv) { + if (icmp6_payload) { + free(icmp6_payload); + icmp6_payload = NULL; + } + + return EXIT_SUCCESS; +} + +static int icmp6_echo_gw_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth6_header(eth_header, src, gw); + + struct ip6_hdr *ip6_header = (struct ip6_hdr *) (ð_header[1]); + uint16_t payload_len = sizeof(struct icmp6_hdr) + icmp6_payload_len; + make_ip6_header(ip6_header, IPPROTO_ICMPV6, payload_len); + + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + make_icmp6_header(icmp6_header); + + char *payload = (char *) icmp6_header + ICMP6_MINLEN; + memcpy(payload, icmp6_payload, icmp6_payload_len); + + return EXIT_SUCCESS; +} + +static int icmp6_echo_gw_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + UNUSED port_h_t dst_port, uint8_t ttl, + UNUSED int probe_num, UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) (ð_header[1]); + struct icmp6_hdr * icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, 0, validation); + + uint16_t icmp6_idnum = icmp_get_idnum(validation); + uint16_t icmp6_seqnum = icmp_get_seqnum(validation); + + uint8_t *ip6_src = (uint8_t *) &(ip6_header->ip6_src); + uint8_t *ip6_dst = (uint8_t *) &(ip6_header->ip6_dst); + for (int i = 0; i < 16; i++) { + ip6_src[i] = src_ip[i]; + ip6_dst[i] = dst_ip[i]; + } + ip6_header->ip6_hlim = ttl; + + icmp6_header->icmp6_id = icmp6_idnum; + icmp6_header->icmp6_seq = icmp6_seqnum; + + icmp6_header->icmp6_cksum = 0; + icmp6_header->icmp6_cksum = + (uint16_t) icmp6_checksum((struct in6_addr *) &(ip6_header->ip6_src), + (struct in6_addr *) &(ip6_header->ip6_dst), + icmp6_header, icmp6_payload_len); + + return EXIT_SUCCESS; +} + +static void icmp6_echo_gw_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) ð_header[1]; + struct icmp6_hdr * icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + + fprintf_eth_header(fp, eth_header); + fprintf_ip6_header(fp, ip6_header); + fprintf(fp, + "ICMPv6\n" + "\tType(1B)\t\t: %u\n" + "\tCode(1B)\t\t: %u\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tIdentifier(2B)\t\t: n:%d (0x%04x) h:%d (0x%04x)\n" + "\tSequence number(2B)\t: n:%d (0x%04x) h:%d (0x%04x)\n", + icmp6_header->icmp6_type, icmp6_header->icmp6_code, + icmp6_header->icmp6_cksum, icmp6_header->icmp6_id, + icmp6_header->icmp6_id, ntohs(icmp6_header->icmp6_id), + ntohs(icmp6_header->icmp6_id), icmp6_header->icmp6_seq, + icmp6_header->icmp6_seq, ntohs(icmp6_header->icmp6_seq), + ntohs(icmp6_header->icmp6_seq)); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int icmp6_echo_gw_validate_packet(const struct ip *ip_hdr, uint32_t len, + int *is_repeat) { + struct ip6_hdr *ip6_header = (struct ip6_hdr *) ip_hdr; + + if (ip6_header->ip6_nxt != IPPROTO_ICMPV6) { + return 0; + } + // IPv6 header is fixed length at 40 bytes + ICMPv6 header + if ((sizeof(struct ip6_hdr) + sizeof(struct icmp6_hdr) + + icmp6_payload_len) > len) { + // buffer not large enough to contain expected icmp header + return 0; + } + + // ICMPv6 header + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + uint16_t icmp6_idnum = icmp6_header->icmp6_id; + uint16_t icmp6_seqnum = icmp6_header->icmp6_seq; + + // ICMPv6 validation is tricky: for some packet types, we must look inside + // the payload + uint8_t validation[VALIDATE_BYTES]; + if (icmp6_header->icmp6_type == ICMP6_TIME_EXCEEDED || + icmp6_header->icmp6_type == ICMP6_DST_UNREACH || + icmp6_header->icmp6_type == ICMP6_PACKET_TOO_BIG || + icmp6_header->icmp6_type == ICMP6_PARAM_PROB) { + + // IPv6 + ICMP6 headers + inner headers + payload (validation) + if (2 * sizeof(struct ip6_hdr) + 2 * sizeof(struct icmp6_hdr) + + icmp6_payload_len > + len) { + return 0; + } + + // Use inner headers for validation + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + struct icmp6_hdr *icmp6_inner_header = + (struct icmp6_hdr *) &ip6_inner_header[1]; + + // Regenerate validation and icmpv6 id based off inner payload + icmp6_idnum = icmp6_inner_header->icmp6_id; + icmp6_seqnum = icmp6_inner_header->icmp6_seq; + + // Send original src and dst IP as data in ICMPv6 payload and regenerate + // the validation here + validate_gen((const uint8_t *) &(ip6_inner_header->ip6_src), + (const uint8_t *) &(ip6_inner_header->ip6_dst), 0, + validation); + } else if (icmp6_header->icmp6_type == ICMP6_ECHO_REPLY) { + validate_gen((const uint8_t *) &(ip6_header->ip6_dst), + (const uint8_t *) &(ip6_header->ip6_src), 0, validation); + } + + // validate icmp id and seqnum + if (icmp6_idnum != icmp_get_idnum(validation)) { + return 0; + } + if (icmp6_seqnum != icmp_get_seqnum(validation)) { + return 0; + } + + // whether repeat: reply ip + char ip_port_str[xconf.ipv46_bytes]; + memcpy(ip_port_str, (char *) &(ip6_header->ip6_src), xconf.ipv46_bytes); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes); + *is_repeat = 0; + } + + return 1; +} + +static void icmp6_echo_gw_process_packet(const u_char *packet, uint32_t len, + fieldset_t * fs, + UNUSED struct timespec ts) { + struct ip6_hdr *ip6_header = + (struct ip6_hdr *) &packet[sizeof(struct ether_header)]; + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + uint32_t hdrlen = sizeof(struct ether_header) + sizeof(struct ip6_hdr) + + 4; // after checksum + + if (icmp6_header->icmp6_type == ICMP6_ECHO_REPLY) { + fs_add_uint64(fs, "success", 0); + fs_add_string(fs, "clas", (char *) "echoreply", 0); + fs_add_string(fs, "desc", (char *) "no code", 0); + } else { + // Use inner IPv6 header values for unsuccessful ICMPv6 replies + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + fs_modify_string( + fs, "saddr", + make_ipv6_str((struct in6_addr *) &(ip6_inner_header->ip6_dst)), 1); + + fs_add_uint64(fs, "success", 1); + fs_add_string(fs, "clas", + (char *) get_icmp6_type_str(icmp6_header->icmp6_type), 0); + fs_add_string(fs, "desc", + (char *) get_icmp6_type_code_str( + icmp6_header->icmp6_type, icmp6_header->icmp6_code), + 0); + } + + fs_add_uint64(fs, "type", icmp6_header->icmp6_type); + fs_add_uint64(fs, "code", icmp6_header->icmp6_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp6_header->icmp6_id)); + fs_add_uint64(fs, "seq", ntohs(icmp6_header->icmp6_seq)); + fs_add_string(fs, "outersaddr", + make_ipv6_str((struct in6_addr *) &(ip6_header->ip6_src)), 1); + + int datalen = len - hdrlen; + + if (datalen > 0) { + const uint8_t *data = (uint8_t *) &packet[hdrlen]; + fs_add_binary(fs, "data", (size_t) datalen, (void *) data, 0); + } else { + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "success", + .type = "int", + .desc = "did probe module classify response as a reply not from target " + "itself"}, + {.name = "clas", + .type = "string", + .desc = "packet classification(type str):\n" + "\t\t\te.g., `echoreply', `other'\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "desc", + .type = "string", + .desc = "ICMPv6 message detail(code str):\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "type", .type = "int", .desc = "ICMPv6 message type"}, + {.name = "code", .type = "int", .desc = "ICMPv6 message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "ICMPv6 id number"}, + {.name = "seq", .type = "int", .desc = "ICMPv6 sequence number"}, + {.name = "outersaddr", + .type = "string", + .desc = "outer src address of ICMPv6 reply packet"}, + {.name = "data", .type = "binary", .desc = "ICMPv6 payload"}, +}; + +probe_module_t module_icmp6_echo_gw = { + .ipv46_flag = 6, + .name = "icmp_echo_gw", + .packet_length = 14 + 40 + 8 + 8, + .pcap_filter = "icmp6 && (ip6[40] == 129 || ip6[40] == 3 || ip6[40] == 1 " + "|| ip6[40] == 2 || ip6[40] == 4)", + .pcap_snaplen = 14 + 2 * (40 + 8) + 8, + .port_args = 0, + .global_init = &icmp6_echo_gw_global_init, + .close = &icmp6_echo_gw_global_cleanup, + .thread_init = &icmp6_echo_gw_thread_init, + .make_packet = &icmp6_echo_gw_make_packet, + .print_packet = &icmp6_echo_gw_print_packet, + .process_packet = &icmp6_echo_gw_process_packet, + .validate_packet = &icmp6_echo_gw_validate_packet, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = "Probe module that sends ICMPv6 echo requests to hosts for " + "discovering gateway.\n" + "And the following argus should be set:\n" + " --iid-module=low_fill\n" + "Payload of ICMPv6 packets will consist of 8 bytes zero unless " + "you customize it with:\n" + " --probe-args=file:/path_to_payload_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164\n" + " --probe-args=icmp-type-code-str", +}; diff --git a/src/probe_modules/module_icmp6_echo_tmxd.c b/src/probe_modules/module_icmp6_echo_tmxd.c new file mode 100644 index 0000000..fdf20d0 --- /dev/null +++ b/src/probe_modules/module_icmp6_echo_tmxd.c @@ -0,0 +1,393 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMPv6 echo request (ping) scans +// to discover routers with routing loop + +#include "module_icmp6.h" + +probe_module_t module_icmp6_echo_tmxd; + +static const char *icmp6_usage_error = + "unknown ICMPv6 probe specification (expected file:/path or text:STRING or " + "hex:01020304 or icmpv6-type-code-str)"; + +static size_t icmp6_payload_len = 0; +static const size_t icmp6_payload_default_len = 8; +static char * icmp6_payload = NULL; + +static int icmp6_echo_timexceed_global_init(struct state_conf *conf) { + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) { + icmp6_payload = xmalloc(icmp6_payload_default_len); + icmp6_payload_len = icmp6_payload_default_len; + return EXIT_SUCCESS; + } + + if (strcmp(conf->probe_args, "icmp-type-code-str") == 0) { + print_icmp6_type_code_str(); + exit(EXIT_SUCCESS); + } + + char *c = strchr(conf->probe_args, ':'); + if (!c) { + log_error("icmp6", icmp6_usage_error); + return EXIT_FAILURE; + } + ++c; + + if (strncmp(conf->probe_args, "text", 4) == 0) { + icmp6_payload = strdup(c); + icmp6_payload_len = strlen(icmp6_payload); + } else if (strncmp(conf->probe_args, "file", 4) == 0) { + FILE *inp = fopen(c, "rb"); + if (!inp) { + log_error("icmp6", "could not open ICMPv6 data file '%s'", c); + free(icmp6_payload); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_END)) { + log_error("icmp6", "unable to get size of ICMPv6 data file '%s'", + c); + free(icmp6_payload); + return EXIT_FAILURE; + } + size_t input_size = ftell(inp); + if (input_size > ICMP6_MAX_PAYLOAD_LEN) { + log_error( + "icmp6", + "input file larger than %d bytes and will not fit on the wire " + "(%llu bytes provided)", + ICMP6_MAX_PAYLOAD_LEN, input_size); + free(icmp6_payload); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_SET)) { + log_error("icmp6", "unable to read ICMPv6 data file '%s'", c); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload = xmalloc(ICMP6_MAX_PAYLOAD_LEN); + icmp6_payload_len = fread(icmp6_payload, 1, ICMP6_MAX_PAYLOAD_LEN, inp); + fclose(inp); + } else if (strncmp(conf->probe_args, "hex", 3) == 0) { + if (strlen(c) % 2 != 0) { + log_error("icmp6", + "invalid hex input (length must be a multiple of 2)"); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload_len = strlen(c) / 2; + icmp6_payload = xmalloc(icmp6_payload_len); + + unsigned int n; + for (size_t i = 0; i < icmp6_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_error("icmp6", "non-hex character: '%c'", c[i * 2]); + free(icmp6_payload); + return EXIT_FAILURE; + } + icmp6_payload[i] = (char) (n & 0xff); + } + } else { + log_error("icmp6", icmp6_usage_error); + free(icmp6_payload); + return EXIT_FAILURE; + } + + if (icmp6_payload_len > ICMP6_MAX_PAYLOAD_LEN) { + log_error("icmp6", + "reducing ICMPv6 payload must be at most %d bytes to fit on " + "the wire (%d were provided)\n", + ICMP6_MAX_PAYLOAD_LEN, icmp6_payload_len); + free(icmp6_payload); + return EXIT_FAILURE; + } + + module_icmp6_echo_tmxd.packet_length = sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + + ICMP6_MINLEN + icmp6_payload_len; + assert(module_icmp6_echo_tmxd.packet_length <= + 1500 + sizeof(struct ether_header)); + module_icmp6_echo_tmxd.pcap_snaplen = + sizeof(struct ether_header) + + 2 * (sizeof(struct ip6_hdr) + ICMP6_MINLEN) + icmp6_payload_len; + + return EXIT_SUCCESS; +} + +static int + icmp6_echo_timexceed_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv) { + if (icmp6_payload) { + free(icmp6_payload); + icmp6_payload = NULL; + } + + return EXIT_SUCCESS; +} + +static int icmp6_echo_timexceed_thread_init(void *buf, macaddr_t *src, + macaddr_t * gw, + UNUSED void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth6_header(eth_header, src, gw); + + struct ip6_hdr *ip6_header = (struct ip6_hdr *) (ð_header[1]); + uint16_t payload_len = sizeof(struct icmp6_hdr) + icmp6_payload_len; + make_ip6_header(ip6_header, IPPROTO_ICMPV6, payload_len); + + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + make_icmp6_header(icmp6_header); + + char *payload = (char *) icmp6_header + ICMP6_MINLEN; + memcpy(payload, icmp6_payload, icmp6_payload_len); + + return EXIT_SUCCESS; +} + +static int icmp6_echo_timexceed_make_packet( + void *buf, UNUSED size_t *buf_len, ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + UNUSED port_h_t dst_port, uint8_t ttl, int probe_num, UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) (ð_header[1]); + struct icmp6_hdr * icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, 0, validation); + + uint16_t icmp6_idnum = icmp_get_idnum(validation); + uint16_t icmp6_seqnum = icmp_get_seqnum(validation); + + uint8_t *ip6_src = (uint8_t *) &(ip6_header->ip6_src); + uint8_t *ip6_dst = (uint8_t *) &(ip6_header->ip6_dst); + for (int i = 0; i < 16; i++) { + ip6_src[i] = src_ip[i]; + ip6_dst[i] = dst_ip[i]; + } + ip6_header->ip6_hlim = ttl - probe_num; + + icmp6_header->icmp6_id = icmp6_idnum; + icmp6_header->icmp6_seq = icmp6_seqnum; + + icmp6_header->icmp6_cksum = 0; + icmp6_header->icmp6_cksum = + (uint16_t) icmp6_checksum((struct in6_addr *) &(ip6_header->ip6_src), + (struct in6_addr *) &(ip6_header->ip6_dst), + icmp6_header, icmp6_payload_len); + + return EXIT_SUCCESS; +} + +static void icmp6_echo_timexceed_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) ð_header[1]; + struct icmp6_hdr * icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + + fprintf_eth_header(fp, eth_header); + fprintf_ip6_header(fp, ip6_header); + fprintf(fp, + "ICMPv6\n" + "\tType(1B)\t\t: %u\n" + "\tCode(1B)\t\t: %u\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tIdentifier(2B)\t\t: n:%d (0x%04x) h:%d (0x%04x)\n" + "\tSequence number(2B)\t: n:%d (0x%04x) h:%d (0x%04x)\n", + icmp6_header->icmp6_type, icmp6_header->icmp6_code, + icmp6_header->icmp6_cksum, icmp6_header->icmp6_id, + icmp6_header->icmp6_id, ntohs(icmp6_header->icmp6_id), + ntohs(icmp6_header->icmp6_id), icmp6_header->icmp6_seq, + icmp6_header->icmp6_seq, ntohs(icmp6_header->icmp6_seq), + ntohs(icmp6_header->icmp6_seq)); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int icmp6_echo_timexceed_validate_packet(const struct ip *ip_hdr, + uint32_t len, int *is_repeat) { + struct ip6_hdr *ip6_header = (struct ip6_hdr *) ip_hdr; + + if (ip6_header->ip6_nxt != IPPROTO_ICMPV6) { + return 0; + } + // IPv6 header is fixed length at 40 bytes + ICMPv6 header + if ((sizeof(struct ip6_hdr) + sizeof(struct icmp6_hdr) + + icmp6_payload_len) > len) { + // buffer not large enough to contain expected icmp header + return 0; + } + + // ICMPv6 header + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + uint16_t icmp6_idnum = icmp6_header->icmp6_id; + uint16_t icmp6_seqnum = icmp6_header->icmp6_seq; + + // ICMPv6 validation is tricky: for some packet types, we must look inside + // the payload + uint8_t validation[VALIDATE_BYTES]; + if (icmp6_header->icmp6_type == ICMP6_TIME_EXCEEDED || + icmp6_header->icmp6_type == ICMP6_DST_UNREACH || + icmp6_header->icmp6_type == ICMP6_PACKET_TOO_BIG || + icmp6_header->icmp6_type == ICMP6_PARAM_PROB) { + + // IPv6 + ICMP6 headers + inner headers + payload (validation) + if (2 * sizeof(struct ip6_hdr) + 2 * sizeof(struct icmp6_hdr) + + icmp6_payload_len > + len) { + return 0; + } + + // Use inner headers for validation + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + struct icmp6_hdr *icmp6_inner_header = + (struct icmp6_hdr *) &ip6_inner_header[1]; + + // Regenerate validation and icmpv6 id based off inner payload + icmp6_idnum = icmp6_inner_header->icmp6_id; + icmp6_seqnum = icmp6_inner_header->icmp6_seq; + + // Send original src and dst IP as data in ICMPv6 payload and regenerate + // the validation here + validate_gen((const uint8_t *) &(ip6_inner_header->ip6_src), + (const uint8_t *) &(ip6_inner_header->ip6_dst), 0, + validation); + } else if (icmp6_header->icmp6_type == ICMP6_ECHO_REPLY) { + validate_gen((const uint8_t *) &(ip6_header->ip6_dst), + (const uint8_t *) &(ip6_header->ip6_src), 0, validation); + } + + // validate icmp id and seqnum + if (icmp6_idnum != icmp_get_idnum(validation)) { + return 0; + } + if (icmp6_seqnum != icmp_get_seqnum(validation)) { + return 0; + } + + // whether repeat: reply + char ip_port_str[xconf.ipv46_bytes + 1]; + memcpy(ip_port_str, (char *) &(ip6_header->ip6_src), xconf.ipv46_bytes); + ip_port_str[xconf.ipv46_bytes] = (char) icmp6_header->icmp6_type; + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 1) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 1); + *is_repeat = 0; + } + + return 1; +} + +static void icmp6_echo_timexceed_process_packet(const u_char *packet, + uint32_t len, fieldset_t *fs, + UNUSED struct timespec ts) { + struct ip6_hdr *ip6_header = + (struct ip6_hdr *) &packet[sizeof(struct ether_header)]; + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + uint32_t hdrlen = sizeof(struct ether_header) + sizeof(struct ip6_hdr) + + 4; // after checksum + + if (icmp6_header->icmp6_type == ICMP6_ECHO_REPLY) { + fs_add_uint64(fs, "success", 0); + fs_add_string(fs, "clas", (char *) "echoreply", 0); + fs_add_string(fs, "desc", (char *) "no code", 0); + } else { + // Use inner IPv6 header values for unsuccessful ICMPv6 replies + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + fs_modify_string( + fs, "saddr", + make_ipv6_str((struct in6_addr *) &(ip6_inner_header->ip6_dst)), 1); + + if (icmp6_header->icmp6_type == ICMP6_TIME_EXCEEDED) { + fs_add_uint64(fs, "success", 1); + } else { + fs_add_uint64(fs, "success", 0); + } + fs_add_string(fs, "clas", + (char *) get_icmp6_type_str(icmp6_header->icmp6_type), 0); + fs_add_string(fs, "desc", + (char *) get_icmp6_type_code_str( + icmp6_header->icmp6_type, icmp6_header->icmp6_code), + 0); + } + + fs_add_uint64(fs, "type", icmp6_header->icmp6_type); + fs_add_uint64(fs, "code", icmp6_header->icmp6_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp6_header->icmp6_id)); + fs_add_uint64(fs, "seq", ntohs(icmp6_header->icmp6_seq)); + fs_add_string(fs, "outersaddr", + make_ipv6_str((struct in6_addr *) &(ip6_header->ip6_src)), 1); + + int datalen = len - hdrlen; + + if (datalen > 0) { + const uint8_t *data = (uint8_t *) &packet[hdrlen]; + fs_add_binary(fs, "data", (size_t) datalen, (void *) data, 0); + } else { + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "success", + .type = "int", + .desc = "did probe module classify response as a time-exceeded reply"}, + {.name = "clas", + .type = "string", + .desc = "packet classification(type str):\n" + "\t\t\te.g., `echoreply', `other'\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "desc", + .type = "string", + .desc = "ICMPv6 message detail(code str):\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "type", .type = "int", .desc = "ICMPv6 message type"}, + {.name = "code", .type = "int", .desc = "ICMPv6 message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "ICMPv6 id number"}, + {.name = "seq", .type = "int", .desc = "ICMPv6 sequence number"}, + {.name = "outersaddr", + .type = "string", + .desc = "outer src address of ICMPv6 reply packet"}, + {.name = "data", .type = "binary", .desc = "ICMPv6 payload"}, +}; + +probe_module_t module_icmp6_echo_tmxd = { + .ipv46_flag = 6, + .name = "icmp_echo_tmxd", + .packet_length = 14 + 40 + 8 + 8, + .pcap_filter = "icmp6 && (ip6[40] == 129 || ip6[40] == 3 || ip6[40] == 1 " + "|| ip6[40] == 2 || ip6[40] == 4)", + .pcap_snaplen = 14 + 2 * (40 + 8) + 8, + .port_args = 0, + .global_init = &icmp6_echo_timexceed_global_init, + .close = &icmp6_echo_timexceed_global_cleanup, + .thread_init = &icmp6_echo_timexceed_thread_init, + .make_packet = &icmp6_echo_timexceed_make_packet, + .print_packet = &icmp6_echo_timexceed_print_packet, + .process_packet = &icmp6_echo_timexceed_process_packet, + .validate_packet = &icmp6_echo_timexceed_validate_packet, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = "Probe module that sends ICMPv6 echo requests to hosts for " + "discovering routers with routing loop. " + "And the following argus should be set:\n" + " --iid-module=rand\n" + " --iid-num=2\n" + " --probe-ttl=64 (max)\n" + " --probes=2 (2 packets with ttl=64,63)\n" + " (`repeat' key: )\n" + "Payload of ICMPv6 packets will consist of 8 bytes zero unless " + "you customize it with:\n" + " --probe-args=file:/path_to_payload_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164\n" + " --probe-args=icmp-type-code-str", +}; diff --git a/src/probe_modules/module_icmp_echo.c b/src/probe_modules/module_icmp_echo.c new file mode 100644 index 0000000..7cb678c --- /dev/null +++ b/src/probe_modules/module_icmp_echo.c @@ -0,0 +1,385 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing ICMP echo request (ping) scans + +#include +#include +#include +#include +#include +#include + +#include "packet.h" +#include "packet_icmp.h" +#include "probe_modules.h" + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../../lib/types.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" +#include "../validate.h" + +#define ICMP_SMALLEST_SIZE 5 +#define ICMP_MAX_PAYLOAD_LEN 1500 - 20 - 8 // 1500 - IP_h - ICMP_h +#define ICMP_TIMXCEED_UNREACH_HEADER_SIZE 8 + +probe_module_t module_icmp_echo; + +const char *icmp_usage_error = + "unknown ICMP probe specification (expected file:/path or text:STRING or " + "hex:01020304 or icmp-type-code-str)"; + +static size_t icmp_payload_len = 0; +static const size_t icmp_payload_default_len = 8; +static char * icmp_payload = NULL; + +static int icmp_echo_global_init(struct state_conf *conf) { + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) { + icmp_payload = xmalloc(icmp_payload_default_len); + icmp_payload_len = icmp_payload_default_len; + return EXIT_SUCCESS; + } + + if (strcmp(conf->probe_args, "icmp-type-code-str") == 0) { + print_icmp_type_code_str(); + exit(EXIT_SUCCESS); + } + + char *c = strchr(conf->probe_args, ':'); + if (!c) { + log_error("icmp", icmp_usage_error); + return EXIT_FAILURE; + } + ++c; + + if (strncmp(conf->probe_args, "text", 4) == 0) { + icmp_payload = strdup(c); + icmp_payload_len = strlen(icmp_payload); + } else if (strncmp(conf->probe_args, "file", 4) == 0) { + FILE *inp = fopen(c, "rb"); + if (!inp) { + log_error("icmp", "could not open ICMP data file '%s'", c); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_END)) { + log_error("icmp", "unable to get size of ICMP data file '%s'", c); + return EXIT_FAILURE; + } + size_t input_size = ftell(inp); + if (input_size > ICMP_MAX_PAYLOAD_LEN) { + log_error( + "icmp", + "input file larger than %d bytes and will not fit on the wire " + "(%llu bytes provided)", + ICMP_MAX_PAYLOAD_LEN, input_size); + return EXIT_FAILURE; + } + if (fseek(inp, 0, SEEK_SET)) { + log_error("icmp", "unable to read ICMP data file '%s'", c); + return EXIT_FAILURE; + } + icmp_payload = xmalloc(ICMP_MAX_PAYLOAD_LEN); + icmp_payload_len = fread(icmp_payload, 1, ICMP_MAX_PAYLOAD_LEN, inp); + fclose(inp); + } else if (strncmp(conf->probe_args, "hex", 3) == 0) { + if (strlen(c) % 2 != 0) { + log_error("icmp", + "invalid hex input (length must be a multiple of 2)"); + return EXIT_FAILURE; + } + icmp_payload_len = strlen(c) / 2; + icmp_payload = xmalloc(icmp_payload_len); + + unsigned int n; + for (size_t i = 0; i < icmp_payload_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + free(icmp_payload); + log_error("icmp", "non-hex character: '%c'", c[i * 2]); + return EXIT_FAILURE; + } + icmp_payload[i] = (char) (n & 0xff); + } + } else { + log_error("icmp", icmp_usage_error); + return EXIT_FAILURE; + } + + if (icmp_payload_len > ICMP_MAX_PAYLOAD_LEN) { + log_error("icmp", + "reducing ICMP payload must be at most %d bytes to fit on " + "the wire (%d were provided)\n", + ICMP_MAX_PAYLOAD_LEN, icmp_payload_len); + return EXIT_FAILURE; + } + + module_icmp_echo.packet_length = sizeof(struct ether_header) + + sizeof(struct ip) + ICMP_MINLEN + + icmp_payload_len; + assert(module_icmp_echo.packet_length <= + 1500 + sizeof(struct ether_header)); + module_icmp_echo.pcap_snaplen = sizeof(struct ether_header) + + 2 * (sizeof(struct ip) + ICMP_MINLEN) + + icmp_payload_len; + + return EXIT_SUCCESS; +} + +static int icmp_echo_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv) { + if (icmp_payload) { + free(icmp_payload); + icmp_payload = NULL; + } + + return EXIT_SUCCESS; +} + +static int icmp_echo_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *) (ð_header[1]); + uint16_t ip_len = sizeof(struct ip) + ICMP_MINLEN + icmp_payload_len; + make_ip_header(ip_header, IPPROTO_ICMP, ip_len); + + struct icmp *icmp_header = (struct icmp *) (&ip_header[1]); + make_icmp_header(icmp_header); + + char *payload = (char *) icmp_header + ICMP_MINLEN; + memcpy(payload, icmp_payload, icmp_payload_len); + + return EXIT_SUCCESS; +} + +static int icmp_echo_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + UNUSED port_h_t dst_port, uint8_t ttl, + UNUSED int probe_num, UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip * ip_header = (struct ip *) (ð_header[1]); + struct icmp * icmp_header = (struct icmp *) (&ip_header[1]); + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, 0, validation); + + uint16_t icmp_idnum = icmp_get_idnum(validation); + uint16_t icmp_seqnum = icmp_get_seqnum(validation); + + ip_header->ip_src.s_addr = *(uint32_t *) src_ip; + ip_header->ip_dst.s_addr = *(uint32_t *) dst_ip; + ip_header->ip_ttl = ttl; + + icmp_header->icmp_id = icmp_idnum; + icmp_header->icmp_seq = icmp_seqnum; + + icmp_header->icmp_cksum = 0; + icmp_header->icmp_cksum = icmp_checksum(icmp_header, icmp_payload_len); + + ip_header->ip_sum = 0; + ip_header->ip_sum = ip_checksum_((unsigned short *) ip_header); + + return EXIT_SUCCESS; +} + +static void icmp_echo_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip * ip_header = (struct ip *) ð_header[1]; + struct icmp * icmp_header = (struct icmp *) (&ip_header[1]); + + fprintf_eth_header(fp, eth_header); + fprintf_ip_header(fp, ip_header); + fprintf(fp, + "ICMP\n" + "\tType(1B)\t\t: %u\n" + "\tCode(1B)\t\t: %u\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tIdentifier(2B)\t\t: n:%d (0x%04x) h:%d (0x%04x)\n" + "\tSequence number(2B)\t: n:%d (0x%04x) h:%d (0x%04x)\n", + icmp_header->icmp_type, icmp_header->icmp_code, + icmp_header->icmp_cksum, icmp_header->icmp_id, icmp_header->icmp_id, + ntohs(icmp_header->icmp_id), ntohs(icmp_header->icmp_id), + icmp_header->icmp_seq, icmp_header->icmp_seq, + ntohs(icmp_header->icmp_seq), ntohs(icmp_header->icmp_seq)); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int icmp_echo_validate_packet(const struct ip *ip_hdr, uint32_t len, + int *is_repeat) { + if (ip_hdr->ip_p != IPPROTO_ICMP) { + return 0; + } + // check if buffer is large enough to contain expected icmp header + if (((uint32_t) 4 * ip_hdr->ip_hl + ICMP_SMALLEST_SIZE) > len) { + return 0; + } + + struct icmp *icmp_header = + (struct icmp *) ((char *) ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t icmp_idnum = icmp_header->icmp_id; + uint16_t icmp_seqnum = icmp_header->icmp_seq; + + // ICMP validation is tricky: for some packet types, we must look inside + // the payload + uint8_t validation[VALIDATE_BYTES]; + if (icmp_header->icmp_type == ICMP_TIMXCEED || + icmp_header->icmp_type == ICMP_UNREACH || + icmp_header->icmp_type == ICMP_PARAMPROB) { + + // Should have 16B TimeExceeded/Dest_Unreachable header + + // original IP header + 1st 8B of original ICMP frame + if ((4 * ip_hdr->ip_hl + ICMP_TIMXCEED_UNREACH_HEADER_SIZE + + sizeof(struct ip)) > len) { + return 0; + } + + struct ip *ip_inner_header = + (struct ip *) ((char *) icmp_header + + ICMP_TIMXCEED_UNREACH_HEADER_SIZE); + if (((uint32_t) 4 * ip_hdr->ip_hl + ICMP_TIMXCEED_UNREACH_HEADER_SIZE + + 4 * ip_inner_header->ip_hl + 8 /*1st 8 bytes of original*/) > + len) { + return 0; + } + + struct icmp *icmp_inner_header = + (struct icmp *) ((char *) ip_inner_header + 4 * ip_hdr->ip_hl); + // Regenerate validation and icmp id based off inner payload + icmp_idnum = icmp_inner_header->icmp_id; + icmp_seqnum = icmp_inner_header->icmp_seq; + + validate_gen((uint8_t *) &(ip_inner_header->ip_src.s_addr), + (uint8_t *) &(ip_inner_header->ip_dst.s_addr), 0, + validation); + } else if (icmp_header->icmp_type == ICMP_ECHOREPLY) { + validate_gen((uint8_t *) &(ip_hdr->ip_dst.s_addr), + (uint8_t *) &(ip_hdr->ip_src.s_addr), 0, validation); + } + + // validate icmp id and seqnum + if (icmp_idnum != icmp_get_idnum(validation)) { + return 0; + } + if (icmp_seqnum != icmp_get_seqnum(validation)) { + return 0; + } + + // whether repeat: reply ip + char ip_port_str[xconf.ipv46_bytes]; + memcpy(ip_port_str, (char *) &(ip_hdr->ip_src.s_addr), xconf.ipv46_bytes); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes); + *is_repeat = 0; + } + + return 1; +} + +static void icmp_echo_process_packet(const u_char *packet, uint32_t len, + fieldset_t * fs, + UNUSED struct timespec ts) { + struct ip * ip_header = (struct ip *) &packet[sizeof(struct ether_header)]; + struct icmp *icmp_header = + (struct icmp *) ((char *) ip_header + 4 * ip_header->ip_hl); + uint32_t hdrlen = sizeof(struct ether_header) + 4 * ip_header->ip_hl + + 4; // after checksum + + if (icmp_header->icmp_type == ICMP_ECHOREPLY) { + fs_add_uint64(fs, "success", 1); + fs_add_string(fs, "clas", (char *) "echoreply", 0); + fs_add_string(fs, "desc", (char *) "no code", 0); + } else { + // Use inner IP header values for unsuccessful ICMP replies + struct ip *ip_inner_header = + (struct ip *) ((char *) icmp_header + + ICMP_TIMXCEED_UNREACH_HEADER_SIZE); + fs_modify_string(fs, "saddr", + make_ip_str(ip_inner_header->ip_dst.s_addr), 1); + + fs_add_bool(fs, "success", 0); + fs_add_string(fs, "clas", + (char *) get_icmp_type_str(icmp_header->icmp_type), 0); + fs_add_string(fs, "desc", + (char *) get_icmp_type_code_str(icmp_header->icmp_type, + icmp_header->icmp_code), + 0); + } + + fs_add_uint64(fs, "type", icmp_header->icmp_type); + fs_add_uint64(fs, "code", icmp_header->icmp_code); + fs_add_uint64(fs, "icmp_id", ntohs(icmp_header->icmp_id)); + fs_add_uint64(fs, "seq", ntohs(icmp_header->icmp_seq)); + fs_add_string(fs, "outersaddr", make_ip_str(ip_header->ip_src.s_addr), 1); + + int datalen = len - hdrlen; + + if (datalen > 0) { + const uint8_t *data = (uint8_t *) &packet[hdrlen]; + fs_add_binary(fs, "data", (size_t) datalen, (void *) data, 0); + } else { + fs_add_null(fs, "data"); + } +} + +static fielddef_t fields[] = { + {.name = "success", + .type = "bool", + .desc = "did probe module classify response as success"}, + {.name = "clas", + .type = "string", + .desc = "packet classification (type str):\n" + "\t\t\te.g., `echoreply', `other'\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "desc", + .type = "string", + .desc = "ICMP message detail(code str):\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "type", .type = "int", .desc = "ICMP message type"}, + {.name = "code", .type = "int", .desc = "ICMP message sub type code"}, + {.name = "icmp_id", .type = "int", .desc = "ICMP id number"}, + {.name = "seq", .type = "int", .desc = "ICMP sequence number"}, + {.name = "outersaddr", + .type = "string", + .desc = "outer src address of icmp reply packet"}, + {.name = "data", .type = "binary", .desc = "ICMP payload"}}; + +probe_module_t module_icmp_echo = { + .ipv46_flag = 4, + .name = "icmp_echo", + .packet_length = 14 + 20 + 8 + 8, + .pcap_filter = "icmp and icmp[0]!=8", + .pcap_snaplen = 14 + 2 * (20 + 8) + 8, + .port_args = 0, + .global_init = &icmp_echo_global_init, + .close = &icmp_echo_global_cleanup, + .thread_init = &icmp_echo_thread_init, + .make_packet = &icmp_echo_make_packet, + .print_packet = &icmp_echo_print_packet, + .process_packet = &icmp_echo_process_packet, + .validate_packet = &icmp_echo_validate_packet, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = "Probe module that sends ICMP echo requests to hosts.\n" + "Payload of ICMP packets will consist of 8 bytes zero unless " + "you customize it with:\n" + " --probe-args=file:/path_to_payload_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164\n" + " --probe-args=icmp-type-code-str", +}; diff --git a/src/probe_modules/module_tcp6_syn.c b/src/probe_modules/module_tcp6_syn.c new file mode 100644 index 0000000..18775b6 --- /dev/null +++ b/src/probe_modules/module_tcp6_syn.c @@ -0,0 +1,226 @@ +/* + * ZMapv6 Copyright 2016 Chair of Network Architectures and Services + * Technical University of Munich + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../../lib/logger.h" +#include "../fieldset.h" +#include "packet.h" +#include "probe_modules.h" +#include "validate.h" + +probe_module_t module_tcp6_syn; +static uint32_t tcp6_num_sports; + +static int tcp6_syn_global_init(struct state_conf *state) { + tcp6_num_sports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int tcp6_syn_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth6_header(eth_header, src, gw); + + struct ip6_hdr *ip6_header = (struct ip6_hdr *) (ð_header[1]); + uint16_t payload_len = sizeof(struct tcphdr); + make_ip6_header(ip6_header, IPPROTO_TCP, payload_len); + + struct tcphdr *tcp_header = (struct tcphdr *) (&ip6_header[1]); + make_tcp_header(tcp_header, TH_SYN); + + return EXIT_SUCCESS; +} + +static int tcp6_syn_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + port_h_t dst_port, uint8_t ttl, int probe_num, + UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) (ð_header[1]); + struct tcphdr * tcp_header = (struct tcphdr *) (&ip6_header[1]); + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, dst_port, validation); + uint32_t tcp_seq = tcp_get_seqnum(validation); + + uint8_t *ip6_src = (uint8_t *) &(ip6_header->ip6_src); + uint8_t *ip6_dst = (uint8_t *) &(ip6_header->ip6_dst); + for (int i = 0; i < 16; i++) { + ip6_src[i] = src_ip[i]; + ip6_dst[i] = dst_ip[i]; + } + + ip6_header->ip6_hlim = ttl; + tcp_header->th_sport = + htons(get_src_port(tcp6_num_sports, probe_num, validation)); + tcp_header->th_dport = htons(dst_port); + tcp_header->th_seq = tcp_seq; + tcp_header->th_sum = 0; + tcp_header->th_sum = + tcp6_checksum((struct in6_addr *) &(ip6_header->ip6_src), + (struct in6_addr *) &(ip6_header->ip6_dst), tcp_header, + sizeof(struct tcphdr)); + + return EXIT_SUCCESS; +} + +// not static because used by synack scan +void tcp6_syn_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) ð_header[1]; + struct tcphdr * tcp_header = (struct tcphdr *) &ip6_header[1]; + + fprintf_eth_header(fp, eth_header); + fprintf_ip6_header(fp, ip6_header); + fprintf(fp, + "TCP\n" + "\tSource Port(2B)\t\t: %u\n" + "\tDestination Port(2B)\t: %u\n" + "\tSequence number(4B)\t: %u\n" + "\tAcknowledge number(4B)\t: %d\n" + "\tHeader Length(4b)\t: %d\n" + "\tFlag(12b)\t\t: 0x%03x\n" + "\tWindow size value(2B)\t: %d\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tUrgent pointer(2B)\t: %d\n", + ntohs(tcp_header->th_sport), ntohs(tcp_header->th_dport), + ntohl(tcp_header->th_seq), ntohl(tcp_header->th_ack), + tcp_header->th_off * 4, tcp_header->th_flags, + ntohs(tcp_header->th_win), tcp_header->th_sum, tcp_header->th_urp); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int tcp6_syn_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED int *is_repeat) { + struct ip6_hdr *ip6_header = (struct ip6_hdr *) ip_hdr; + + if (ip6_header->ip6_nxt != IPPROTO_TCP) { + return PACKET_INVALID; + } + if ((sizeof(struct ip6_hdr) + sizeof(struct tcphdr)) > len) { + // buffer not large enough to contain expected tcp_header header + return PACKET_INVALID; + } + + struct tcphdr *tcp_header = (struct tcphdr *) (&ip6_header[1]); + uint16_t dport = ntohs(tcp_header->th_sport); + uint16_t sport = ntohs(tcp_header->th_dport); + + // validate source port + if (!xconf.target_port_flag[dport]) { + return PACKET_INVALID; + } + + // validate destination port + uint8_t validation[VALIDATE_BYTES]; + validate_gen((uint8_t *) &(ip6_header->ip6_dst), + (uint8_t *) &(ip6_header->ip6_src), dport, validation); + if (!check_src_port(sport, tcp6_num_sports, validation)) { + return PACKET_INVALID; + } + + // We treat RST packets different from non RST packets + uint32_t tcp_seq = tcp_get_seqnum(validation); + if (tcp_header->th_flags & TH_RST) { + // For RST packets, recv(ack) == sent(seq) + 0 or + 1 + if (htonl(tcp_header->th_ack) != htonl(tcp_seq) && + htonl(tcp_header->th_ack) != htonl(tcp_seq) + 1) { + return PACKET_INVALID; + } + } else { + // For non RST packets, recv(ack) == sent(seq) + 1 + if (htonl(tcp_header->th_ack) != htonl(tcp_seq) + 1) { + return PACKET_INVALID; + } + } + + // whether repeat: reply ip + dport + char ip_port_str[xconf.ipv46_bytes + 2]; + memcpy(ip_port_str, (char *) &(ip6_header->ip6_src), xconf.ipv46_bytes); + ip_port_str[xconf.ipv46_bytes] = (char) (dport >> 8u); + ip_port_str[xconf.ipv46_bytes + 1] = (char) (dport & 0xffu); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2); + *is_repeat = 0; + } + return PACKET_VALID; +} + +static void tcp6_syn_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED struct timespec ts) { + struct ip6_hdr *ip6_header = + (struct ip6_hdr *) &packet[sizeof(struct ether_header)]; + struct tcphdr *tcp_header = (struct tcphdr *) (&ip6_header[1]); + + fs_add_uint64(fs, "sport", (uint64_t) ntohs(tcp_header->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t) ntohs(tcp_header->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t) ntohl(tcp_header->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t) ntohl(tcp_header->th_ack)); + fs_add_uint64(fs, "window", (uint64_t) ntohs(tcp_header->th_win)); + + if (tcp_header->th_flags & TH_RST) { // RST packet + fs_add_string(fs, "clas", (char *) "rst", 0); + fs_add_bool(fs, "success", 0); + } else { // SYNACK packet + fs_add_string(fs, "clas", (char *) "synack", 0); + fs_add_bool(fs, "success", 1); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window size"}, + {.name = "clas", + .type = "string", + .desc = "packet classification" + "\t\t\te.g, `synack', `rst'\n"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}}; + +probe_module_t module_tcp6_syn = { + .ipv46_flag = 6, + .name = "tcp_syn", + .packet_length = 14 + 40 + 20, + .pcap_filter = "ip6 proto 6 && (ip6[53] & 4 != 0 || ip6[53] == 18)", + .pcap_snaplen = 14 + 40 + 20 + 40, + .port_args = 1, + .global_init = &tcp6_syn_global_init, + .thread_init = &tcp6_syn_thread_init, + .make_packet = &tcp6_syn_make_packet, + .print_packet = &tcp6_syn_print_packet, + .process_packet = &tcp6_syn_process_packet, + .validate_packet = &tcp6_syn_validate_packet, + .close = NULL, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response."}; diff --git a/src/probe_modules/module_tcp_syn.c b/src/probe_modules/module_tcp_syn.c new file mode 100644 index 0000000..98df2c3 --- /dev/null +++ b/src/probe_modules/module_tcp_syn.c @@ -0,0 +1,221 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +// probe module for performing TCP SYN scans + +#include +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "../fieldset.h" +#include "packet.h" +#include "probe_modules.h" +#include "validate.h" + +probe_module_t module_tcp_syn; +static uint32_t tcp_num_sports; + +static int tcp_syn_global_init(struct state_conf *state) { + tcp_num_sports = state->source_port_last - state->source_port_first + 1; + return EXIT_SUCCESS; +} + +static int tcp_syn_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, + UNUSED void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *) (ð_header[1]); + uint16_t ip_len = sizeof(struct ip) + sizeof(struct tcphdr); + make_ip_header(ip_header, IPPROTO_TCP, ip_len); + + struct tcphdr *tcp_header = (struct tcphdr *) (&ip_header[1]); + make_tcp_header(tcp_header, TH_SYN); + + return EXIT_SUCCESS; +} + +static int tcp_syn_make_packet(void *buf, UNUSED size_t *buf_len, + ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + port_h_t dst_port, uint8_t ttl, int probe_num, + UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip * ip_header = (struct ip *) (ð_header[1]); + struct tcphdr * tcp_header = (struct tcphdr *) (&ip_header[1]); + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, dst_port, validation); + uint32_t tcp_seq = tcp_get_seqnum(validation); + + ip_header->ip_src.s_addr = *(uint32_t *) src_ip; + ip_header->ip_dst.s_addr = *(uint32_t *) dst_ip; + ip_header->ip_ttl = ttl; + tcp_header->th_sport = + htons(get_src_port(tcp_num_sports, probe_num, validation)); + tcp_header->th_dport = htons(dst_port); + tcp_header->th_seq = tcp_seq; + tcp_header->th_sum = 0; + tcp_header->th_sum = + tcp_checksum(sizeof(struct tcphdr), ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, tcp_header); + + ip_header->ip_sum = 0; + ip_header->ip_sum = ip_checksum_((unsigned short *) ip_header); + + return EXIT_SUCCESS; +} + +// not static because used by synack scan +void tcp_syn_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip * ip_header = (struct ip *) ð_header[1]; + struct tcphdr * tcp_header = (struct tcphdr *) &ip_header[1]; + + fprintf_eth_header(fp, eth_header); + fprintf_ip_header(fp, ip_header); + fprintf(fp, + "TCP\n" + "\tSource Port(2B)\t\t: %u\n" + "\tDestination Port(2B)\t: %u\n" + "\tSequence number(4B)\t: %u\n" + "\tAcknowledge number(4B)\t: %d\n" + "\tHeader Length(4b)\t: %d\n" + "\tFlag(12b)\t\t: 0x%03x\n" + "\tWindow size value(2B)\t: %d\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tUrgent pointer(2B)\t: %d\n", + ntohs(tcp_header->th_sport), ntohs(tcp_header->th_dport), + ntohl(tcp_header->th_seq), ntohl(tcp_header->th_ack), + tcp_header->th_off * 4, tcp_header->th_flags, + ntohs(tcp_header->th_win), tcp_header->th_sum, tcp_header->th_urp); + fprintf(fp, "------------------------------------------------------\n"); +} + +static int tcp_syn_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED int *is_repeat) { + if (ip_hdr->ip_p != IPPROTO_TCP) { + return PACKET_INVALID; + } + if ((4 * ip_hdr->ip_hl + sizeof(struct tcphdr)) > len) { + // buffer not large enough to contain expected tcp_header header + return PACKET_INVALID; + } + + struct tcphdr *tcp_header = + (struct tcphdr *) ((char *) ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t dport = ntohs(tcp_header->th_sport); + uint16_t sport = ntohs(tcp_header->th_dport); + + // validate destination port + if (!xconf.target_port_flag[dport]) { + return PACKET_INVALID; + } + + // validate source port + uint8_t validation[VALIDATE_BYTES]; + validate_gen((uint8_t *) &(ip_hdr->ip_dst.s_addr), + (uint8_t *) &(ip_hdr->ip_src.s_addr), dport, validation); + if (!check_src_port(sport, tcp_num_sports, validation)) { + return PACKET_INVALID; + } + + // We treat RST packets different from non RST packets + uint32_t tcp_seq = tcp_get_seqnum(validation); + if (tcp_header->th_flags & TH_RST) { + // For RST packets, recv(ack) == sent(seq) + 0 or + 1 + if (htonl(tcp_header->th_ack) != htonl(tcp_seq) && + htonl(tcp_header->th_ack) != htonl(tcp_seq) + 1) { + return PACKET_INVALID; + } + } else { + // For non RST packets, recv(ack) == sent(seq) + 1 + if (htonl(tcp_header->th_ack) != htonl(tcp_seq) + 1) { + return PACKET_INVALID; + } + } + + // whether repeat: reply ip + dport + char ip_port_str[xconf.ipv46_bytes + 2]; + memcpy(ip_port_str, (char *) &(ip_hdr->ip_src.s_addr), xconf.ipv46_bytes); + ip_port_str[xconf.ipv46_bytes] = (char) (dport >> 8u); + ip_port_str[xconf.ipv46_bytes + 1] = (char) (dport & 0xffu); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2); + *is_repeat = 0; + } + + return PACKET_VALID; +} + +static void tcp_syn_process_packet(const u_char *packet, UNUSED uint32_t len, + fieldset_t *fs, UNUSED struct timespec ts) { + struct ip *ip_heaher = (struct ip *) &packet[sizeof(struct ether_header)]; + struct tcphdr *tcp_header = + (struct tcphdr *) ((char *) ip_heaher + 4 * ip_heaher->ip_hl); + + fs_add_uint64(fs, "sport", (uint64_t) ntohs(tcp_header->th_sport)); + fs_add_uint64(fs, "dport", (uint64_t) ntohs(tcp_header->th_dport)); + fs_add_uint64(fs, "seqnum", (uint64_t) ntohl(tcp_header->th_seq)); + fs_add_uint64(fs, "acknum", (uint64_t) ntohl(tcp_header->th_ack)); + fs_add_uint64(fs, "window", (uint64_t) ntohs(tcp_header->th_win)); + + if (tcp_header->th_flags & TH_RST) { // RST packet + fs_add_string(fs, "clas", (char *) "rst", 0); + fs_add_bool(fs, "success", 0); + } else { // SYNACK packet + fs_add_string(fs, "clas", (char *) "synack", 0); + fs_add_bool(fs, "success", 1); + } +} + +static fielddef_t fields[] = { + {.name = "sport", .type = "int", .desc = "TCP source port"}, + {.name = "dport", .type = "int", .desc = "TCP destination port"}, + {.name = "seqnum", .type = "int", .desc = "TCP sequence number"}, + {.name = "acknum", .type = "int", .desc = "TCP acknowledgement number"}, + {.name = "window", .type = "int", .desc = "TCP window size"}, + {.name = "clas", + .type = "string", + .desc = "packet classification" + "\t\t\te.g., `synack', `rst'\n"}, + {.name = "success", + .type = "bool", + .desc = "is response considered success"}}; + +probe_module_t module_tcp_syn = { + .ipv46_flag = 4, + .name = "tcp_syn", + .packet_length = 14 + 20 + 20, + .pcap_filter = "ip && tcp && (tcp[13] & 4 != 0 || tcp[13] == 18)", + .pcap_snaplen = 14 + 20 + 20 + 40, + .port_args = 1, + .global_init = &tcp_syn_global_init, + .thread_init = &tcp_syn_thread_init, + .make_packet = &tcp_syn_make_packet, + .print_packet = &tcp_syn_print_packet, + .process_packet = &tcp_syn_process_packet, + .validate_packet = &tcp_syn_validate_packet, + .close = NULL, + .output_type = OUTPUT_TYPE_STATIC, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = "Probe module that sends a TCP SYN packet to a specific " + "port. Possible classifications are: synack and rst. A " + "SYN-ACK packet is considered a success and a reset packet " + "is considered a failed response."}; diff --git a/src/probe_modules/module_udp.c b/src/probe_modules/module_udp.c new file mode 100644 index 0000000..0733dad --- /dev/null +++ b/src/probe_modules/module_udp.c @@ -0,0 +1,1070 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* send module for performing arbitrary IPv4 UDP scans */ + +#include +#include +#include +#include +#include +#include +#include + +#include "../../lib/blocklist.h" +#include "../../lib/lockfd.h" +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "../aesrand.h" +#include "../state.h" +#include "packet.h" +#include "packet_icmp.h" +#include "probe_modules.h" +#include "validate.h" + +#include "module_udp.h" + +#define MAX_UDP_PAYLOAD_LEN 1500 - 20 - 8 // 1500 - IP_h - UDP_h +#define ICMP_UNREACH_HEADER_SIZE 8 +#define UDP_SEND_MSG_EXT_NUM 3 + +static char * udp_send_msg = NULL; +static int udp_send_msg_len = 0; +static int udp_send_substitutions = 0; +static udp_payload_template_t *udp_template = NULL; + +static char * udp_send_msg_list[MAX_PORT_NUM]; +static int udp_send_msg_len_list[MAX_PORT_NUM]; +static const char *udp_send_msg_ext_list[UDP_SEND_MSG_EXT_NUM] = {"pkt", "txt", + "hex"}; +enum UDP_SEND_MSG_EXT { PKT, TXT, HEX }; +static int udp_send_dir_payload = 0; + +static const char *udp_send_msg_default = + "GET / HTTP/1.1\r\nHost: ida.\r\n\r\n"; +static const int udp_send_msg_default_len = 30; + +const char *udp_usage_error = "unknown UDP probe specification " + "(expected file:/path or text:STRING or " + "hex:01020304 or dir:/dir_to_files or " + "template:/path or template-fields or " + "icmp-type-code-str)"; + +const unsigned char *charset_alphanum = + (unsigned char + *) "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +const unsigned char *charset_alpha = + (unsigned char *) "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const unsigned char *charset_digit = (unsigned char *) "0123456789"; +const unsigned char charset_all[257] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, + 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, + 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, + 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, + 0xfd, 0xfe, 0xff, 0x00}; + +static int udp_num_sports; + +probe_module_t module_udp; + +// Field definitions for template parsing and displaying usage +static udp_payload_field_type_def_t udp_payload_template_fields[] = { + {.name = "SADDR_N", + .ftype = UDP_SADDR_N, + .desc = "Source IPv4 address in network byte order"}, + {.name = "SADDR", + .ftype = UDP_SADDR_A, + .desc = "Source IPv4 address in dotted-quad format"}, + {.name = "DADDR_N", + .ftype = UDP_DADDR_N, + .desc = "Destination IPv4 address in network byte order"}, + {.name = "DADDR", + .ftype = UDP_DADDR_A, + .desc = "Destination IPv4 address in dotted-quad format"}, + {.name = "SPORT_N", + .ftype = UDP_SPORT_N, + .desc = "UDP source port in netowrk byte order"}, + {.name = "SPORT", + .ftype = UDP_SPORT_A, + .desc = "UDP source port in ascii format"}, + {.name = "DPORT_N", + .ftype = UDP_DPORT_N, + .desc = "UDP destination port in network byte order"}, + {.name = "DPORT", + .ftype = UDP_DPORT_A, + .desc = "UDP destination port in ascii format"}, + {.name = "RAND_BYTE", + .ftype = UDP_RAND_BYTE, + .desc = "Random bytes from 0-255"}, + {.name = "RAND_DIGIT", + .ftype = UDP_RAND_DIGIT, + .desc = "Random digits from 0-9"}, + {.name = "RAND_ALPHA", + .ftype = UDP_RAND_ALPHA, + .desc = "Random mixed-case letters (a-z)"}, + {.name = "RAND_ALPHANUM", + .ftype = UDP_RAND_ALPHANUM, + .desc = "Random mixed-case letters (a-z) and numbers"}}; +static uint32_t udp_num_template_field_types = + sizeof(udp_payload_template_fields) / + sizeof(udp_payload_template_fields[0]); + +void udp_set_num_sports(int x) { udp_num_sports = x; } + +static int udp_load_payload(const char *dir) { + log_debug("udp", "load udp payload from dir"); + if (!opendir(dir)) { + log_error("udp", "could not read UDP data dir '%s'", dir); + return EXIT_FAILURE; + } + + udp_send_msg_len = 0; + + char dir_new[strlen(dir) + 1]; + strncpy(dir_new, dir, strlen(dir)); + if (dir[strlen(dir) - 1] != '/') strcat(dir_new, "/"); + + FILE * inp; + unsigned int n, f; + int port; + + for (int i = 0; i < xconf.target_port_num; i++) { + port = xconf.target_port_list[i]; + f = 0; + udp_send_msg_list[port] = strdup(udp_send_msg_default); + udp_send_msg_len_list[port] = udp_send_msg_default_len; + char *new_file = xmalloc(strlen(dir_new) + 10); + for (int j = 0; j < UDP_SEND_MSG_EXT_NUM; j++) { + memset(new_file, 0, strlen(dir_new) + 10); + sprintf(new_file, "%s%d.%s", dir_new, port, + udp_send_msg_ext_list[j]); + + switch (j) { + case PKT: + inp = fopen(new_file, "rb"); + break; + case TXT: + case HEX: + inp = fopen(new_file, "r"); + break; + default: + continue; + } + if (!inp) continue; + + char *udp_send_msg_t; + int udp_send_msg_len_t; + free(udp_send_msg_list[port]); + switch (j) { + case PKT: + case TXT: + udp_send_msg_list[port] = xmalloc(MAX_UDP_PAYLOAD_LEN); + udp_send_msg_len_list[port] = (int) fread( + udp_send_msg_list[port], 1, MAX_UDP_PAYLOAD_LEN, inp); + fclose(inp); + break; + case HEX: + udp_send_msg_t = xmalloc(MAX_UDP_PAYLOAD_LEN * 2); + udp_send_msg_len_t = (int) fread(udp_send_msg_t, 1, + MAX_UDP_PAYLOAD_LEN * 2, inp); + fclose(inp); + udp_send_msg_len_t = (int) udp_send_msg_len_t / 2; + udp_send_msg_list[port] = xmalloc(udp_send_msg_len_t); + udp_send_msg_len_list[port] = udp_send_msg_len_t; + for (int k = 0; k < udp_send_msg_len_t; k++) { + if (sscanf(udp_send_msg_t + (k * 2), "%2x", &n) != 1) { + log_error("udp", "non-hex character: '%02x'", + udp_send_msg_t[k * 2]); + free(udp_send_msg_t); + return EXIT_FAILURE; + } + udp_send_msg_list[port][k] = (n & 0xff); + } + free(udp_send_msg_t); + break; + default: + break; + } + + j = UDP_SEND_MSG_EXT_NUM; + f = 1; + log_debug("udp", "read UDP data file '%s' for port %d", new_file, + port); + } + + if (sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + udp_send_msg_len_list[port] > + MAX_PACKET_SIZE) { + log_error("udp", "payload too long, should be no more than %d", + MAX_PACKET_SIZE - + (sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr))); + return EXIT_FAILURE; + } + + udp_send_msg_len += udp_send_msg_len_list[port]; + + free(new_file); + if (!f) { + log_debug("udp", "no UDP data file for port %d, using default", + port); + } + } + + udp_send_msg_len /= xconf.target_port_num; + log_debug("udp", "load udp payload from dir completed"); + + return EXIT_SUCCESS; +} + +int udp_global_init(struct state_conf *conf) { + udp_num_sports = conf->source_port_last - conf->source_port_first + 1; + + udp_send_msg = strdup(udp_send_msg_default); + udp_send_msg_len = udp_send_msg_default_len; + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) + return EXIT_SUCCESS; + + char * args, *c; + int i; + unsigned int n; + + FILE *inp; + + args = strdup(conf->probe_args); + assert(args); + + if (strcmp(args, "icmp-type-code-str") == 0) { + print_icmp_type_code_str(); + exit(EXIT_SUCCESS); + } + + if (strcmp(args, "template-fields") == 0) { + lock_file(stderr); + fprintf(stderr, "%s", "List of allowed UDP template fields\n"); + fprintf(stderr, + "------------------------------------------------------\n"); + fprintf(stderr, "%-15s: %s\n", "Name", "Description"); + fprintf(stderr, + "------------------------------------------------------\n"); + for (uint32_t i = 0; i < udp_num_template_field_types; i++) { + fprintf(stderr, "%-15s: %s\n", udp_payload_template_fields[i].name, + udp_payload_template_fields[i].desc); + } + fprintf(stderr, + "------------------------------------------------------\n"); + fprintf(stderr, "%s\n", + "Example:\n Specify the field at ${}, e.g., ${SADDR}, " + "${RAND_DIGIT=9}"); + unlock_file(stderr); + exit(EXIT_SUCCESS); + } + + c = strchr(args, ':'); + if (!c) { + free(args); + free(udp_send_msg); + log_error("udp", udp_usage_error); + return EXIT_FAILURE; + } + + *c++ = 0; + + if (strcmp(args, "text") == 0) { + free(udp_send_msg); + udp_send_msg = strdup(c); + udp_send_msg_len = (int) strlen(udp_send_msg); + } else if (strcmp(args, "file") == 0 || strcmp(args, "template") == 0) { + inp = fopen(c, "rb"); + if (!inp) { + log_error("udp", "could not open UDP data file '%s'", c); + free(args); + free(udp_send_msg); + return EXIT_FAILURE; + } + free(udp_send_msg); + udp_send_msg = xmalloc(MAX_UDP_PAYLOAD_LEN); + udp_send_msg_len = + (int) fread(udp_send_msg, 1, MAX_UDP_PAYLOAD_LEN, inp); + fclose(inp); + + if (strcmp(args, "template") == 0) { + udp_send_substitutions = 1; + udp_template = udp_template_load(udp_send_msg, udp_send_msg_len); + } + + } else if (strcmp(args, "hex") == 0) { + udp_send_msg_len = (int) strlen(c) / 2; + free(udp_send_msg); + udp_send_msg = xmalloc(udp_send_msg_len); + + for (i = 0; i < udp_send_msg_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_error("udp", "non-hex character: '%c'", c[i * 2]); + free(args); + free(udp_send_msg); + return EXIT_FAILURE; + } + udp_send_msg[i] = (n & 0xff); + } + } else if (strcmp(args, "dir") == 0) { + udp_send_dir_payload = 1; + if (udp_load_payload(c)) + return EXIT_FAILURE; // udp_send_msg_len = avg len + } else { + log_error("udp", udp_usage_error); + free(args); + free(udp_send_msg); + return EXIT_FAILURE; + } + + if (udp_send_msg_len > MAX_UDP_PAYLOAD_LEN) { + log_warn("udp", + "warning: reducing UDP payload to %d bytes (from %d) to fit " + "on the wire", + MAX_UDP_PAYLOAD_LEN, udp_send_msg_len); + udp_send_msg_len = MAX_UDP_PAYLOAD_LEN; + } + + module_udp.packet_length = sizeof(struct ether_header) + sizeof(struct ip) + + sizeof(struct udphdr) + udp_send_msg_len; + assert(module_udp.packet_length <= MAX_PACKET_SIZE); + + free(args); + + return EXIT_SUCCESS; +} + +int udp_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv) { + if (udp_send_msg) { + free(udp_send_msg); + udp_send_msg = NULL; + } + + if (udp_template) { + udp_template_free(udp_template); + udp_template = NULL; + } + + if (udp_send_dir_payload) { + for (int i = 0; i < MAX_PORT_NUM; i++) + if (udp_send_msg_list[i]) { + free(udp_send_msg_list[i]); + udp_send_msg_list[i] = NULL; + } + } + + return EXIT_SUCCESS; +} + +int udp_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth_header(eth_header, src, gw); + + struct ip *ip_header = (struct ip *) (ð_header[1]); + uint16_t ip_len = + sizeof(struct ip) + sizeof(struct udphdr) + udp_send_msg_len; + make_ip_header(ip_header, IPPROTO_UDP, ip_len); + + struct udphdr *udp_header = (struct udphdr *) (&ip_header[1]); + uint16_t udp_len = sizeof(struct udphdr) + udp_send_msg_len; + make_udp_header(udp_header, udp_len); + + char *payload = (char *) (&udp_header[1]); + memcpy(payload, udp_send_msg, udp_send_msg_len); + + // Seed our random number generator with the global generator + uint32_t seed = aesrand_getword(xconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg_ptr = aes; + + return EXIT_SUCCESS; +} + +int udp_make_packet(void *buf, UNUSED size_t *buf_len, ipaddr_n_t *src_ip, + ipaddr_n_t *dst_ip, port_h_t dst_port, uint8_t ttl, + int probe_num, UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip * ip_header = (struct ip *) (ð_header[1]); + struct udphdr * udp_header = (struct udphdr *) &ip_header[1]; + + ip_header->ip_src.s_addr = *(uint32_t *) src_ip; + ip_header->ip_dst.s_addr = *(uint32_t *) dst_ip; + ip_header->ip_ttl = ttl; + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, dst_port, validation); + + udp_header->uh_sport = + htons(get_src_port(udp_num_sports, probe_num, validation)); + udp_header->uh_dport = htons(dst_port); + + if (udp_send_substitutions) { // from templete + char *payload = (char *) &udp_header[1]; + int payload_len = 0; + memset(payload, 0, MAX_UDP_PAYLOAD_LEN); + + // Grab our random number generator + aesrand_t *aes = (aesrand_t *) arg; + + // The buf is a stack var of our caller of size MAX_PACKET_SIZE + // Recalculate the payload using the loaded template + payload_len = + udp_template_build(udp_template, payload, MAX_UDP_PAYLOAD_LEN, + ip_header, udp_header, aes); + // Recalculate the total length of the packet + module_udp.packet_length = sizeof(struct ether_header) + + sizeof(struct ip) + sizeof(struct udphdr) + + payload_len; + + // If success is zero, the template output was truncated + if (payload_len <= 0) { + log_fatal("udp", "UDP payload template generated an empty payload"); + exit(EXIT_FAILURE); + } + + // Update the IP and UDP headers to match the new payload length + size_t ip_len = sizeof(struct ip) + sizeof(struct udphdr) + payload_len; + ip_header->ip_len = htons(ip_len); + udp_header->uh_ulen = htons(sizeof(struct udphdr) + payload_len); + *buf_len = ip_len + sizeof(struct ether_header); + } else if (udp_send_dir_payload) { // from dir + char *payload = (char *) &udp_header[1]; + int payload_len = udp_send_msg_len_list[dst_port]; + memcpy(payload, udp_send_msg_list[dst_port], payload_len); + + // Update the IP and UDP headers to match the new payload length + size_t ip_len = sizeof(struct ip) + sizeof(struct udphdr) + payload_len; + ip_header->ip_len = htons(ip_len); + udp_header->uh_ulen = htons(sizeof(struct udphdr) + payload_len); + *buf_len = ip_len + sizeof(struct ether_header); + } + + udp_header->uh_sum = 0; + udp_header->uh_sum = udp_checksum(ip_header->ip_src.s_addr, + ip_header->ip_dst.s_addr, udp_header); + + ip_header->ip_sum = 0; + ip_header->ip_sum = ip_checksum_((unsigned short *) ip_header); + + return EXIT_SUCCESS; +} + +void udp_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip * ip_header = (struct ip *) ð_header[1]; + struct udphdr * udp_header = (struct udphdr *) (&ip_header[1]); + + fprintf_eth_header(fp, eth_header); + fprintf_ip_header(fp, ip_header); + fprintf(fp, + "UDP\n" + "\tSource Port(2B)\t\t: %u\n" + "\tDestination Port(2B)\t: %u\n" + "\tLength(2B)\t\t: %u\n" + "\tChecksum(2B)\t\t: 0x%04x\n", + ntohs(udp_header->uh_sport), ntohs(udp_header->uh_dport), + ntohs(udp_header->uh_ulen), ntohs(udp_header->uh_sum)); + fprintf(fp, "------------------------------------------------------\n"); +} + +int udp_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED int *is_repeat) { + uint16_t dport; + if (ip_hdr->ip_p == IPPROTO_UDP) { + if ((4 * ip_hdr->ip_hl + sizeof(struct udphdr)) > len) { + // buffer not large enough to contain expected udp + // header + return PACKET_INVALID; + } + + struct udphdr *udp_header = + (struct udphdr *) ((char *) ip_hdr + 4 * ip_hdr->ip_hl); + uint16_t sport = ntohs(udp_header->uh_dport); + dport = ntohs(udp_header->uh_sport); + + if (!xconf.target_port_flag[dport]) { + return PACKET_INVALID; + } + + uint8_t validation[VALIDATE_BYTES]; + validate_gen((uint8_t *) &(ip_hdr->ip_dst.s_addr), + (uint8_t *) &(ip_hdr->ip_src.s_addr), dport, validation); + if (!check_src_port(sport, udp_num_sports, validation)) { + return PACKET_INVALID; + } + + if (!blocklist_is_allowed_ip((uint8_t *) &(ip_hdr->ip_src.s_addr))) { + return PACKET_INVALID; + } + + } else if (ip_hdr->ip_p == IPPROTO_ICMP) { + // UDP can return ICMP Destination unreach + // IP( ICMP( IP( UDP ) ) ) for a destination unreach + const uint32_t min_len = 4 * ip_hdr->ip_hl + ICMP_UNREACH_HEADER_SIZE + + sizeof(struct ip) + sizeof(struct udphdr); + if (len < min_len) { + // Not enough information for us to validate + return PACKET_INVALID; + } + + struct icmp *icmp_header = + (struct icmp *) ((char *) ip_hdr + 4 * ip_hdr->ip_hl); + if (!(icmp_header->icmp_type == ICMP_TIMXCEED || + icmp_header->icmp_type == ICMP_UNREACH || + icmp_header->icmp_type == ICMP_PARAMPROB)) { + return PACKET_INVALID; + } + + struct ip *ip_inner_header = + (struct ip *) ((char *) icmp_header + ICMP_UNREACH_HEADER_SIZE); + // Now we know the actual inner ip length, we should recheck the + // buffer + if (len < 4 * ip_inner_header->ip_hl - sizeof(struct ip) + min_len) { + return PACKET_INVALID; + } + + // This is the UDP packet we sent + struct udphdr *udp_inner_header = + (struct udphdr *) ((char *) ip_inner_header + + 4 * ip_inner_header->ip_hl); + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port + dport = ntohs(udp_inner_header->uh_dport); + uint16_t sport = ntohs(udp_inner_header->uh_sport); + if (!xconf.target_port_flag[dport]) { + return PACKET_INVALID; + } + + uint8_t validation[VALIDATE_BYTES]; + validate_gen((uint8_t *) &(ip_inner_header->ip_src.s_addr), + (uint8_t *) &(ip_inner_header->ip_dst.s_addr), dport, + validation); + if (!check_src_port(sport, udp_num_sports, validation)) { + return PACKET_INVALID; + } + + // find original destination IP and check that we sent a packet + // to that IP address + if (!blocklist_is_allowed_ip( + (uint8_t *) &(ip_inner_header->ip_dst.s_addr))) { + return PACKET_INVALID; + } + + } else { + return PACKET_INVALID; + } + + // whether repeat: reply ip + dport + char ip_port_str[xconf.ipv46_bytes + 2]; + memcpy(ip_port_str, (char *) &(ip_hdr->ip_src.s_addr), xconf.ipv46_bytes); + ip_port_str[xconf.ipv46_bytes] = (char) (dport >> 8u); + ip_port_str[xconf.ipv46_bytes + 1] = (char) (dport & 0xffu); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2); + *is_repeat = 0; + } + + return PACKET_VALID; +} + +void udp_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + UNUSED struct timespec ts) { + struct ip *ip_header = (struct ip *) &packet[sizeof(struct ether_header)]; + if (ip_header->ip_p == IPPROTO_UDP) { + struct udphdr *udp_header = + (struct udphdr *) ((char *) ip_header + ip_header->ip_hl * 4); + fs_add_bool(fs, "success", 1); + fs_add_constchar(fs, "clas", "udp"); + fs_add_uint64(fs, "sport", ntohs(udp_header->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp_header->uh_dport)); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_str"); + fs_add_uint64(fs, "udp_pkt_size", ntohs(udp_header->uh_ulen)); + // Verify that the UDP length is big enough for the header and + // at least one byte + uint16_t payload_len = + ntohs(udp_header->uh_ulen - sizeof(struct udphdr)); + if (payload_len > 0) { + uint32_t max_rlen = len - sizeof(struct ether_header) - + ip_header->ip_hl * 4 - sizeof(struct udphdr); + uint32_t max_ilen = ntohs(ip_header->ip_len) - + ip_header->ip_hl * 4 - sizeof(struct udphdr); + + // Verify that the UDP length is inside of our received + // buffer + if (payload_len > max_rlen) { + payload_len = max_rlen; + } + // Verify that the UDP length is inside of our IP packet + if (payload_len > max_ilen) { + payload_len = max_ilen; + } + fs_add_binary(fs, "data", payload_len, (void *) &udp_header[1], 0); + // Some devices reply with a zero UDP length but still + // return data, ignore the data + } else { + fs_add_null(fs, "data"); + } + } else if (ip_header->ip_p == IPPROTO_ICMP) { + struct icmp *icmp_header = + (struct icmp *) ((char *) ip_header + ip_header->ip_hl * 4); + struct ip *ip_inner_header = + (struct ip *) ((char *) icmp_header + ICMP_UNREACH_HEADER_SIZE); + struct udphdr *udp_inner_header = + (struct udphdr *) (&ip_inner_header[1]); + // ICMP unreach comes from another server (not the one we sent a + // probe to); But we will fix up saddr to be who we sent the + // probe to, in case you care. + fs_modify_string(fs, "saddr", + make_ip_str(ip_inner_header->ip_dst.s_addr), 1); + fs_add_bool(fs, "success", 0); + fs_add_string(fs, "clas", + (char *) get_icmp_type_str(icmp_header->icmp_type), 0); + fs_add_uint64(fs, "sport", ntohs(udp_inner_header->uh_dport)); + fs_add_uint64(fs, "dport", ntohs(udp_inner_header->uh_sport)); + fs_add_string(fs, "icmp_responder", + make_ip_str(ip_header->ip_src.s_addr), 1); + fs_add_uint64(fs, "icmp_type", icmp_header->icmp_type); + fs_add_uint64(fs, "icmp_code", icmp_header->icmp_code); + fs_add_string(fs, "icmp_str", + (char *) get_icmp_type_code_str(icmp_header->icmp_type, + icmp_header->icmp_code), + 0); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + } else { + fs_add_bool(fs, "success", 0); + fs_add_string(fs, "clas", (char *) "other", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_str"); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + } +} + +// +// payload templete +int udp_random_bytes(char *dst, int len, const unsigned char *charset, + int charset_len, aesrand_t *aes) { + int i; + for (i = 0; i < len; i++) { + *dst++ = charset[(aesrand_getword(aes) & 0xFFFFFFFF) % charset_len]; + } + + return i; +} + +int udp_template_build(udp_payload_template_t *t, char *out, unsigned int len, + struct ip *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes) { + udp_payload_field_t *c; + char * p; + char * max; + char tmp[256]; + int full = 0; + unsigned int x, y; + uint32_t * u32; + uint16_t * u16; + + max = out + len; + p = out; + + for (x = 0; x < t->fcount; x++) { + c = t->fields[x]; + + // Exit the processing loop if our packet buffer would overflow + if (p + c->length >= max) { + full = 1; + return 0; + } + + switch (c->ftype) { + + // These fields have a specified output length value + + case UDP_DATA: + if (!(c->data && c->length)) break; + memcpy(p, c->data, c->length); + p += c->length; + break; + + case UDP_RAND_DIGIT: + p += udp_random_bytes(p, c->length, charset_digit, 10, aes); + break; + + case UDP_RAND_ALPHA: + p += udp_random_bytes(p, c->length, charset_alpha, 52, aes); + break; + + case UDP_RAND_ALPHANUM: + p += udp_random_bytes(p, c->length, charset_alphanum, 62, aes); + break; + + case UDP_RAND_BYTE: + p += udp_random_bytes(p, c->length, charset_all, 256, aes); + break; + + // These fields need to calculate size on their own + + // TODO: Condense these case statements to remove redundant code + case UDP_SADDR_A: + if (p + 15 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET, (char *) &ip_hdr->ip_src, tmp, sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP_DADDR_A: + if (p + 15 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET, (char *) &ip_hdr->ip_dst, tmp, sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP_SADDR_N: + if (p + 4 >= max) { + full = 1; + break; + } + + u32 = (uint32_t *) p; + *u32 = ip_hdr->ip_src.s_addr; + p += 4; + break; + + case UDP_DADDR_N: + if (p + 4 >= max) { + full = 1; + break; + } + u32 = (uint32_t *) p; + *u32 = ip_hdr->ip_dst.s_addr; + p += 4; + break; + + case UDP_SPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *) p; + *u16 = udp_hdr->uh_sport; + p += 2; + break; + + case UDP_DPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *) p; + *u16 = udp_hdr->uh_sport; + p += 2; + break; + + case UDP_SPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + + case UDP_DPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + } + + // Bail out if our packet buffer would overflow + if (full == 1) { + return 0; + } + } + + return p - out; +} + +// Free all buffers held by the payload template, including its own +void udp_template_free(udp_payload_template_t *t) { + for (unsigned int x = 0; x < t->fcount; x++) { + if (t->fields[x]->data) { + free(t->fields[x]->data); + t->fields[x]->data = NULL; + } + free(t->fields[x]); + t->fields[x] = NULL; + } + free(t->fields); + t->fields = NULL; + t->fcount = 0; + free(t); +} + +// Add a new field to the template +void udp_template_add_field(udp_payload_template_t * t, + udp_payload_field_type_t ftype, unsigned int length, + char *data) { + udp_payload_field_t *c; + + t->fcount++; + t->fields = xrealloc(t->fields, sizeof(udp_payload_field_t) * t->fcount); + if (!t->fields) { + exit(1); + } + + t->fields[t->fcount - 1] = xmalloc(sizeof(udp_payload_field_t)); + c = t->fields[t->fcount - 1]; + + if (!c) { + exit(1); + } + + c->ftype = ftype; + c->length = length; + c->data = data; +} + +// Convert a string field name to a field type, parsing any specified length +// value +int udp_template_lookup_field(char *vname, udp_payload_field_t *c) { + char * param; + unsigned int f; + unsigned int olen = 0; + unsigned int fcount = sizeof(udp_payload_template_fields) / + sizeof(udp_payload_template_fields[0]); + + param = strstr((const char *) vname, "="); + if (param) { + *param = '\0'; + param++; + } + + // Most field types treat their parameter as a generator output length + // unless it is ignored (ADDR, PORT, etc). + if (param) { + olen = atoi((const char *) param); + } + + // Find a field that matches the + for (f = 0; f < fcount; f++) { + + if (strcmp((char *) vname, udp_payload_template_fields[f].name) == 0) { + c->ftype = udp_payload_template_fields[f].ftype; + c->length = olen; + c->data = NULL; + return 1; + } + } + + // No match, skip and treat it as a data field + return 0; +} + +// Allocate a payload template and populate it by parsing a template file as a +// binary buffer +udp_payload_template_t *udp_template_load(char *buf, unsigned int len) { + udp_payload_template_t *t = xmalloc(sizeof(udp_payload_template_t)); + + // The last $ we encountered outside of a field specifier + char *dollar = NULL; + + // The last { we encountered outside of a field specifier + char *lbrack = NULL; + + // Track the start pointer of a data field (static) + char *s = buf; + + // Track the index into the template + char *p = buf; + + char * tmp; + unsigned int tlen; + + udp_payload_field_t c; + + t->fcount = 0; + t->fields = NULL; + + while (p < (buf + len)) { + switch (*p) { + + case '$': + if ((dollar && !lbrack) || !dollar) { + dollar = p; + } + p++; + continue; + + case '{': + if (dollar && !lbrack) { + lbrack = p; + } + + p++; + continue; + + case '}': + if (!(dollar && lbrack)) { + p++; + continue; + } + + // Store the leading bytes before ${ as a data field + tlen = dollar - s; + if (tlen > 0) { + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp_template_add_field(t, UDP_DATA, tlen, tmp); + } + + tmp = xcalloc(1, p - lbrack); + memcpy(tmp, lbrack + 1, p - lbrack - 1); + + if (udp_template_lookup_field(tmp, &c)) { + udp_template_add_field(t, c.ftype, c.length, c.data); + + // Push the pointer past the } if this was a + // valid variable + s = p + 1; + } else { + + // Rewind back to the ${ sequence if this was an + // invalid variable + s = dollar; + } + + free(tmp); + break; + + default: + if (dollar && lbrack) { + p++; + continue; + } + } + + dollar = NULL; + lbrack = NULL; + + p++; + } + + // Store the trailing bytes as a final data field + if (s < p) { + tlen = p - s; + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp_template_add_field(t, UDP_DATA, tlen, tmp); + } + + return t; +} + +static fielddef_t fields[] = { + {.name = "success", + .type = "bool", + .desc = "is response considered success"}, + {.name = "clas", + .type = "string", + .desc = "packet classification(type str):\n" + "\t\t\te.g., `udp', `unreach', `other'\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + {.name = "icmp_responder", + .type = "string", + .desc = "source IP address of ICMP message"}, + {.name = "icmp_type", .type = "int", .desc = "icmp message type"}, + {.name = "icmp_code", .type = "int", .desc = "icmp message code"}, + {.name = "icmp_str", + .type = "string", + .desc = "ICMP message detail(code str):\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "udp_pkt_size", .type = "int", .desc = "UDP packet length"}, + {.name = "data", .type = "binary", .desc = "UDP payload"}}; + +probe_module_t module_udp = { + .ipv46_flag = 4, + .name = "udp", + .packet_length = 14 + 20 + 8 + 30, + .pcap_filter = "(ip && udp) || icmp", + .pcap_snaplen = 1500, + .port_args = 1, + .global_init = &udp_global_init, + .thread_init = &udp_thread_init, + .make_packet = &udp_make_packet, + .print_packet = &udp_print_packet, + .validate_packet = &udp_validate_packet, + .process_packet = &udp_process_packet, + .close = &udp_global_cleanup, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = + "Probe module that sends UDP packets to hosts.\n" + "Packets can optionally be templated based on destination host.\n" + " --probe-args=file:/path_to_packet_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164\n" + " --probe-args=dir:/dir_to_files\n" + " (each file named by port num, e.g., 53.pkt>txt>hex)\n" + " --probe-args=template:/path_to_template_file\n" + " --probe-args=template-fields (list allowed UDP template fields)\n" + " --probe-args=icmp-type-code-str (list allowed type/code str)", +}; diff --git a/src/probe_modules/module_udp.h b/src/probe_modules/module_udp.h new file mode 100644 index 0000000..c5b2c9e --- /dev/null +++ b/src/probe_modules/module_udp.h @@ -0,0 +1,104 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_UDP_H +#define XMAP_MODULE_UDP_H + +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "aesrand.h" +#include "state.h" +#include "types.h" + +typedef enum udp_payload_field_type { + UDP_DATA, + UDP_SADDR_N, + UDP_SADDR_A, + UDP_DADDR_N, + UDP_DADDR_A, + UDP_SPORT_N, + UDP_SPORT_A, + UDP_DPORT_N, + UDP_DPORT_A, + UDP_RAND_BYTE, + UDP_RAND_DIGIT, + UDP_RAND_ALPHA, + UDP_RAND_ALPHANUM +} udp_payload_field_type_t; + +typedef struct udp_payload_field_type_def { + const char * name; + const char * desc; + udp_payload_field_type_t ftype; +} udp_payload_field_type_def_t; + +typedef struct udp_payload_field { + enum udp_payload_field_type ftype; + unsigned int length; + char * data; +} udp_payload_field_t; + +typedef struct udp_payload_template { + unsigned int fcount; + struct udp_payload_field **fields; +} udp_payload_template_t; + +typedef struct udp_payload_output { + int length; + char *data; +} udp_payload_output_t; + +int udp_global_init(struct state_conf *conf); + +int udp_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv); + +int udp_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, void **arg_ptr); + +int udp_make_packet(void *buf, UNUSED size_t *buf_len, ipaddr_n_t *src_ip, + ipaddr_n_t *dst_ip, port_h_t dst_port, uint8_t ttl, + int probe_num, UNUSED void *arg); + +void udp_print_packet(FILE *fp, void *packet); + +int udp_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED int *is_repeat); + +void udp_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + UNUSED struct timespec ts); + +// other +extern const char *udp_unreach_strings[]; + +void udp_set_num_sports(int x); + +// udp field +int udp_template_build(udp_payload_template_t *t, char *out, unsigned int len, + struct ip *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes); + +void udp_template_free(udp_payload_template_t *t); + +void udp_template_add_field(udp_payload_template_t * t, + udp_payload_field_type_t ftype, unsigned int length, + char *data); + +int udp_template_lookup_field(char *vname, udp_payload_field_t *c); + +udp_payload_template_t *udp_template_load(char *buf, unsigned int len); + +#endif // XMAP_MODULE_UDP_H diff --git a/src/probe_modules/module_udp6.c b/src/probe_modules/module_udp6.c new file mode 100644 index 0000000..1f66148 --- /dev/null +++ b/src/probe_modules/module_udp6.c @@ -0,0 +1,1079 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +/* send module for performing arbitrary IPv6 UDP scans */ + +#include "module_udp6.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../../lib/blocklist.h" +#include "../../lib/includes.h" +#include "../../lib/lockfd.h" +#include "../../lib/logger.h" +#include "../../lib/xalloc.h" +#include "../aesrand.h" +#include "../state.h" +#include "packet.h" +#include "packet_icmp6.h" +#include "probe_modules.h" +#include "validate.h" + +#define MAX_UDP6_PAYLOAD_LEN 1500 - 40 - 8 // 1500 - IPv6_h - UDP_h +#define ICMP6_UNREACH_HEADER_SIZE 8 +#define UDP6_SEND_MSG_EXT_NUM 3 + +static char * udp6_send_msg = NULL; +static int udp6_send_msg_len = 0; +static int udp6_send_substitutions = 0; +static udp6_payload_template_t *udp6_template = NULL; + +static char * udp6_send_msg_list[MAX_PORT_NUM]; +static int udp6_send_msg_len_list[MAX_PORT_NUM]; +static const char *udp6_send_msg_ext_list[UDP6_SEND_MSG_EXT_NUM] = { + "pkt", "txt", "hex"}; +enum UDP6_SEND_MSG_EXT { PKT, TXT, HEX }; +static int udp6_send_dir_payload = 0; + +static const char *udp6_send_msg_default = + "GET / HTTP/1.1\r\nHost: ida.\r\n\r\n"; +static const int udp6_send_msg_default_len = 30; + +const char *udp6_usage_error = "unknown UDP probe specification " + "(expected file:/path or text:STRING or " + "hex:01020304 or dir:/dir_to_files or " + "template:/path or template-fields or " + "icmp-type-code-str)"; + +const unsigned char *charset_alphanum6 = + (unsigned char + *) "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +const unsigned char *charset_alpha6 = + (unsigned char *) "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; +const unsigned char *charset_digit6 = (unsigned char *) "0123456789"; +const unsigned char charset_all6[257] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, + 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, + 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, + 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, + 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, + 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, + 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, + 0xfd, 0xfe, 0xff, 0x00}; + +static int udp6_num_sports; + +probe_module_t module_udp6; + +// Field definitions for template parsing and displaying usage +static udp6_payload_field_type_def_t udp6_payload_template_fields[] = { + {.name = "SADDR_N", + .ftype = UDP6_SADDR_N, + .desc = "Source IPv6 address in network byte order"}, + {.name = "SADDR", + .ftype = UDP6_SADDR_A, + .desc = "Source IPv6 address in dotted-quad format"}, + {.name = "DADDR_N", + .ftype = UDP6_DADDR_N, + .desc = "Destination IPv6 address in network byte order"}, + {.name = "DADDR", + .ftype = UDP6_DADDR_A, + .desc = "Destination IPv6 address in dotted-quad format"}, + {.name = "SPORT_N", + .ftype = UDP6_SPORT_N, + .desc = "UDP source port in netowrk byte order"}, + {.name = "SPORT", + .ftype = UDP6_SPORT_A, + .desc = "UDP source port in ascii format"}, + {.name = "DPORT_N", + .ftype = UDP6_DPORT_N, + .desc = "UDP destination port in network byte order"}, + {.name = "DPORT", + .ftype = UDP6_DPORT_A, + .desc = "UDP destination port in ascii format"}, + {.name = "RAND_BYTE", + .ftype = UDP6_RAND_BYTE, + .desc = "Random bytes from 0-255"}, + {.name = "RAND_DIGIT", + .ftype = UDP6_RAND_DIGIT, + .desc = "Random digits from 0-9"}, + {.name = "RAND_ALPHA", + .ftype = UDP6_RAND_ALPHA, + .desc = "Random mixed-case letters (a-z)"}, + {.name = "RAND_ALPHANUM", + .ftype = UDP6_RAND_ALPHANUM, + .desc = "Random mixed-case letters (a-z) and numbers"}}; +static uint32_t udp6_num_template_field_types = + sizeof(udp6_payload_template_fields) / + sizeof(udp6_payload_template_fields[0]); + +void udp6_set_num_sports(int x) { udp6_num_sports = x; } + +static int udp6_load_payload(const char *dir) { + log_debug("udp6", "load udp payload from dir"); + if (!opendir(dir)) { + log_error("udp6", "could not read UDP data dir '%s'", dir); + return EXIT_FAILURE; + } + + udp6_send_msg_len = 0; + + char dir_new[strlen(dir) + 1]; + strncpy(dir_new, dir, strlen(dir)); + if (dir[strlen(dir) - 1] != '/') dir_new[strlen(dir)] = '/'; + + FILE * inp; + unsigned int n, f; + int port; + + for (int i = 0; i < xconf.target_port_num; i++) { + port = xconf.target_port_list[i]; + f = 0; + udp6_send_msg_list[port] = strdup(udp6_send_msg_default); + udp6_send_msg_len_list[port] = udp6_send_msg_default_len; + char *new_file = xmalloc(strlen(dir_new) + 10); + for (int j = 0; j < UDP6_SEND_MSG_EXT_NUM; j++) { + memset(new_file, 0, strlen(dir_new) + 10); + sprintf(new_file, "%s%d.%s", dir_new, port, + udp6_send_msg_ext_list[j]); + + switch (j) { + case PKT: + inp = fopen(new_file, "rb"); + break; + case TXT: + case HEX: + inp = fopen(new_file, "r"); + break; + default: + continue; + } + if (!inp) continue; + + char *udp6_send_msg_t; + int udp6_send_msg_len_t; + free(udp6_send_msg_list[port]); + switch (j) { + case PKT: + case TXT: + udp6_send_msg_list[port] = xmalloc(MAX_UDP6_PAYLOAD_LEN); + udp6_send_msg_len_list[port] = (int) fread( + udp6_send_msg_list[port], 1, MAX_UDP6_PAYLOAD_LEN, inp); + fclose(inp); + break; + case HEX: + udp6_send_msg_t = xmalloc(MAX_UDP6_PAYLOAD_LEN * 2); + udp6_send_msg_len_t = (int) fread( + udp6_send_msg_t, 1, MAX_UDP6_PAYLOAD_LEN * 2, inp); + fclose(inp); + udp6_send_msg_len_t = (int) udp6_send_msg_len_t / 2; + udp6_send_msg_list[port] = xmalloc(udp6_send_msg_len_t); + udp6_send_msg_len_list[port] = udp6_send_msg_len_t; + for (int k = 0; k < udp6_send_msg_len_t; k++) { + if (sscanf(udp6_send_msg_t + (k * 2), "%2x", &n) != 1) { + log_error("udp6", "non-hex character: '%02x'", + udp6_send_msg_t[k * 2]); + free(udp6_send_msg_t); + return EXIT_FAILURE; + } + udp6_send_msg_list[port][k] = (n & 0xff); + } + free(udp6_send_msg_t); + break; + default: + break; + } + + j = UDP6_SEND_MSG_EXT_NUM; + f = 1; + log_debug("udp6", "read UDP data file '%s' for port %d", new_file, + port); + } + + if (sizeof(struct ether_header) + sizeof(struct ip6_hdr) + + sizeof(struct udphdr) + udp6_send_msg_len_list[port] > + MAX_PACKET_SIZE) { + log_error("udp6", "payload too long, should be no more than %d", + MAX_PACKET_SIZE - + (sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + sizeof(struct udphdr))); + return EXIT_FAILURE; + } + + udp6_send_msg_len += udp6_send_msg_len_list[port]; + + free(new_file); + if (!f) { + log_debug("udp6", "no UDP data file for port %d, using default", + port); + } + } + + udp6_send_msg_len /= xconf.target_port_num; + log_debug("udp6", "load udp payload from dir completed"); + + return EXIT_SUCCESS; +} + +int udp6_global_init(struct state_conf *conf) { + udp6_num_sports = conf->source_port_last - conf->source_port_first + 1; + + udp6_send_msg = strdup(udp6_send_msg_default); + udp6_send_msg_len = udp6_send_msg_default_len; + if (!(conf->probe_args && strlen(conf->probe_args) > 0)) + return EXIT_SUCCESS; + + char * args, *c; + int i; + unsigned int n; + + FILE *inp; + + args = strdup(conf->probe_args); + assert(args); + + if (strcmp(args, "icmp-type-code-str") == 0) { + print_icmp6_type_code_str(); + exit(EXIT_SUCCESS); + } + + if (strcmp(args, "template-fields") == 0) { + lock_file(stderr); + fprintf(stderr, "%s", "List of allowed UDP template fields\n"); + fprintf(stderr, + "------------------------------------------------------\n"); + fprintf(stderr, "%-15s: %s\n", "Name", "Description"); + fprintf(stderr, + "------------------------------------------------------\n"); + for (uint32_t i = 0; i < udp6_num_template_field_types; i++) { + fprintf(stderr, "%-15s: %s\n", udp6_payload_template_fields[i].name, + udp6_payload_template_fields[i].desc); + } + fprintf(stderr, + "------------------------------------------------------\n"); + fprintf(stderr, "%s\n", + "Example:\n Specify the field at ${}, e.g., ${SADDR}, " + "${RAND_DIGIT=9}"); + unlock_file(stderr); + exit(EXIT_SUCCESS); + } + + c = strchr(args, ':'); + if (!c) { + free(args); + free(udp6_send_msg); + log_error("udp6", udp6_usage_error); + return EXIT_FAILURE; + } + + *c++ = 0; + + if (strcmp(args, "text") == 0) { + free(udp6_send_msg); + udp6_send_msg = strdup(c); + udp6_send_msg_len = (int) strlen(udp6_send_msg); + } else if (strcmp(args, "file") == 0 || strcmp(args, "template") == 0) { + inp = fopen(c, "rb"); + if (!inp) { + log_error("udp6", "could not open UDP data file '%s'", c); + free(args); + free(udp6_send_msg); + return EXIT_FAILURE; + } + free(udp6_send_msg); + udp6_send_msg = xmalloc(MAX_UDP6_PAYLOAD_LEN); + udp6_send_msg_len = + (int) fread(udp6_send_msg, 1, MAX_UDP6_PAYLOAD_LEN, inp); + fclose(inp); + + if (strcmp(args, "template") == 0) { + udp6_send_substitutions = 1; + udp6_template = + udp6_template_load(udp6_send_msg, udp6_send_msg_len); + } + + } else if (strcmp(args, "hex") == 0) { + udp6_send_msg_len = (int) strlen(c) / 2; + free(udp6_send_msg); + udp6_send_msg = xmalloc(udp6_send_msg_len); + + for (i = 0; i < udp6_send_msg_len; i++) { + if (sscanf(c + (i * 2), "%2x", &n) != 1) { + log_error("udp6", "non-hex character: '%c'", c[i * 2]); + free(args); + free(udp6_send_msg); + return EXIT_FAILURE; + } + udp6_send_msg[i] = (n & 0xff); + } + } else if (strcmp(args, "dir") == 0) { + udp6_send_dir_payload = 1; + if (udp6_load_payload(c)) + return EXIT_FAILURE; // udp6_send_msg_len = avg len + } else { + free(args); + free(udp6_send_msg); + log_error("udp6", udp6_usage_error); + return EXIT_FAILURE; + } + + if (udp6_send_msg_len > MAX_UDP6_PAYLOAD_LEN) { + log_warn("udp6", + "warning: reducing UDP payload to %d bytes (from %d) to fit " + "on the wire\n", + MAX_UDP6_PAYLOAD_LEN, udp6_send_msg_len); + udp6_send_msg_len = MAX_UDP6_PAYLOAD_LEN; + } + + module_udp6.packet_length = sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + sizeof(struct udphdr) + + udp6_send_msg_len; + assert(module_udp6.packet_length <= MAX_PACKET_SIZE); + + free(args); + + return EXIT_SUCCESS; +} + +int udp6_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv) { + if (udp6_send_msg) { + free(udp6_send_msg); + udp6_send_msg = NULL; + } + + if (udp6_template) { + udp6_template_free(udp6_template); + udp6_template = NULL; + } + + if (udp6_send_dir_payload) { + for (int i = 0; i < MAX_PORT_NUM; i++) + if (udp6_send_msg_list[i]) { + free(udp6_send_msg_list[i]); + udp6_send_msg_list[i] = NULL; + } + } + + return EXIT_SUCCESS; +} + +int udp6_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, void **arg_ptr) { + memset(buf, 0, MAX_PACKET_SIZE); + + struct ether_header *eth_header = (struct ether_header *) buf; + make_eth6_header(eth_header, src, gw); + + struct ip6_hdr *ip6_header = (struct ip6_hdr *) (ð_header[1]); + uint16_t payload_len = sizeof(struct udphdr) + udp6_send_msg_len; + make_ip6_header(ip6_header, IPPROTO_UDP, payload_len); + + struct udphdr *udp6_header = (struct udphdr *) (&ip6_header[1]); + uint16_t udp_len = sizeof(struct udphdr) + udp6_send_msg_len; + make_udp_header(udp6_header, udp_len); + + char *payload = (char *) (&udp6_header[1]); + memcpy(payload, udp6_send_msg, udp6_send_msg_len); + + // Seed our random number generator with the global generator + uint32_t seed = aesrand_getword(xconf.aes); + aesrand_t *aes = aesrand_init_from_seed(seed); + *arg_ptr = aes; + + return EXIT_SUCCESS; +} + +int udp6_make_packet(void *buf, UNUSED size_t *buf_len, ipaddr_n_t *src_ip, + ipaddr_n_t *dst_ip, port_h_t dst_port, uint8_t ttl, + int probe_num, UNUSED void *arg) { + struct ether_header *eth_header = (struct ether_header *) buf; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) (ð_header[1]); + struct udphdr * udp_header = (struct udphdr *) &ip6_header[1]; + + uint8_t validation[VALIDATE_BYTES]; + validate_gen(src_ip, dst_ip, dst_port, validation); + + uint8_t *ip6_src = (uint8_t *) &(ip6_header->ip6_src); + uint8_t *ip6_dst = (uint8_t *) &(ip6_header->ip6_dst); + for (int i = 0; i < 16; i++) { + ip6_src[i] = src_ip[i]; + ip6_dst[i] = dst_ip[i]; + } + ip6_header->ip6_hlim = ttl; + + udp_header->uh_sport = + htons(get_src_port(udp6_num_sports, probe_num, validation)); + udp_header->uh_dport = htons(dst_port); + + // from templete + if (udp6_send_substitutions) { + char *payload = (char *) &udp_header[1]; + int payload_len = 0; + memset(payload, 0, MAX_UDP6_PAYLOAD_LEN); + + // Grab our random number generator + aesrand_t *aes = (aesrand_t *) arg; + + // The buf is a stack var of our caller of size MAX_PACKET_SIZE + // Recalculate the payload using the loaded template + payload_len = + udp6_template_build(udp6_template, payload, MAX_UDP6_PAYLOAD_LEN, + ip6_header, udp_header, aes); + // Recalculate the total length of the packet + module_udp6.packet_length = sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + + sizeof(struct udphdr) + payload_len; + + // If success is zero, the template output was truncated + if (payload_len <= 0) { + log_fatal("udp6", + "UDP payload template generated an empty payload"); + exit(EXIT_FAILURE); + } + + // Update the IPv6 and UDP headers to match the new payload length + size_t udp_len = sizeof(struct udphdr) + payload_len; + ip6_header->ip6_plen = htons(udp_len); + udp_header->uh_ulen = htons(udp_len); + *buf_len = + udp_len + sizeof(struct ip6_hdr) + sizeof(struct ether_header); + } else if (udp6_send_dir_payload) { // from dir + char *payload = (char *) &udp_header[1]; + int payload_len = udp6_send_msg_len_list[dst_port]; + memcpy(payload, udp6_send_msg_list[dst_port], payload_len); + + // Update the IPv6 and UDP headers to match the new payload length + size_t udp_len = sizeof(struct udphdr) + payload_len; + ip6_header->ip6_plen = htons(udp_len); + udp_header->uh_ulen = htons(udp_len); + *buf_len = + udp_len + sizeof(struct ip6_hdr) + sizeof(struct ether_header); + } + + udp_header->uh_sum = 0; + udp_header->uh_sum = + udp6_checksum((struct in6_addr *) &(ip6_header->ip6_src), + (struct in6_addr *) &(ip6_header->ip6_dst), udp_header); + + return EXIT_SUCCESS; +} + +void udp6_print_packet(FILE *fp, void *packet) { + struct ether_header *eth_header = (struct ether_header *) packet; + struct ip6_hdr * ip6_header = (struct ip6_hdr *) ð_header[1]; + struct udphdr * udp_header = (struct udphdr *) (&ip6_header[1]); + + fprintf_eth_header(fp, eth_header); + fprintf_ip6_header(fp, ip6_header); + fprintf(fp, + "UDP\n" + "\tSource Port(2B)\t\t: %u\n" + "\tDestination Port(2B)\t: %u\n" + "\tLength(2B)\t\t: %u\n" + "\tChecksum(2B)\t\t: 0x%04x\n", + ntohs(udp_header->uh_sport), ntohs(udp_header->uh_dport), + ntohs(udp_header->uh_ulen), ntohs(udp_header->uh_sum)); + fprintf(fp, "------------------------------------------------------\n"); +} + +int udp6_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED int *is_repeat) { + struct ip6_hdr *ip6_header = (struct ip6_hdr *) ip_hdr; + uint16_t dport; + + if (ip6_header->ip6_nxt == IPPROTO_UDP) { + if ((sizeof(struct ip6_hdr) + sizeof(struct udphdr)) > len) { + // buffer not large enough to contain expected udp + // header + return PACKET_INVALID; + } + + struct udphdr *udp_header = (struct udphdr *) (&ip6_header[1]); + uint16_t sport = ntohs(udp_header->uh_dport); + dport = ntohs(udp_header->uh_sport); + if (!xconf.target_port_flag[dport]) { + return PACKET_INVALID; + } + + uint8_t validation[VALIDATE_BYTES]; + validate_gen((uint8_t *) &(ip6_header->ip6_dst), + (uint8_t *) &(ip6_header->ip6_src), dport, validation); + if (!check_src_port(sport, udp6_num_sports, validation)) { + return PACKET_INVALID; + } + + if (!blocklist_is_allowed_ip((uint8_t *) &(ip6_header->ip6_src))) { + return PACKET_INVALID; + } + + } else if (ip6_header->ip6_nxt == IPPROTO_ICMPV6) { + // UDP can return ICMPv6 Destination unreach + // IPv6( ICMPv6( IPv6( UDP ) ) ) for a destination unreach + const uint32_t min_len = sizeof(struct ip6_hdr) + + ICMP6_UNREACH_HEADER_SIZE + + sizeof(struct ip6_hdr) + sizeof(struct udphdr); + if (len < min_len) { + // Not enough information for us to validate + return PACKET_INVALID; + } + + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + if (!(icmp6_header->icmp6_type == ICMP6_TIME_EXCEEDED || + icmp6_header->icmp6_type == ICMP6_DST_UNREACH || + icmp6_header->icmp6_type == ICMP6_PACKET_TOO_BIG || + icmp6_header->icmp6_type == ICMP6_PARAM_PROB)) { + return PACKET_INVALID; + } + + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + // find original destination IPv6 and check that we sent a packet + // to that IPv6 address + + // This is the UDP packet we sent + struct udphdr *udp_inner_header = + (struct udphdr *) (&ip6_inner_header[1]); + // we can always check the destination port because this is the + // original packet and wouldn't have been altered by something + // responding on a different port + dport = ntohs(udp_inner_header->uh_dport); + uint16_t sport = ntohs(udp_inner_header->uh_sport); + if (!xconf.target_port_flag[dport]) { + return PACKET_INVALID; + } + + uint8_t validation[VALIDATE_BYTES]; + validate_gen((uint8_t *) &(ip6_inner_header->ip6_src), + (uint8_t *) &(ip6_inner_header->ip6_dst), dport, + validation); + if (!check_src_port(sport, udp6_num_sports, validation)) { + return PACKET_INVALID; + } + + if (!blocklist_is_allowed_ip( + (uint8_t *) &(ip6_inner_header->ip6_dst))) { + return PACKET_INVALID; + } + + } else { + return PACKET_INVALID; + } + + // whether repeat: reply ip + dport + char ip_port_str[xconf.ipv46_bytes + 2]; + memcpy(ip_port_str, (char *) &(ip6_header->ip6_src), xconf.ipv46_bytes); + ip_port_str[xconf.ipv46_bytes] = (char) (dport >> 8u); + ip_port_str[xconf.ipv46_bytes + 1] = (char) (dport & 0xffu); + if (bloom_filter_check_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2) == BLOOM_FAILURE) { + bloom_filter_add_string(&xrecv.bf, (const char *) ip_port_str, + xconf.ipv46_bytes + 2); + *is_repeat = 0; + } + + return PACKET_VALID; +} + +void udp6_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + UNUSED struct timespec ts) { + struct ip6_hdr *ip6_header = + (struct ip6_hdr *) &packet[sizeof(struct ether_header)]; + if (ip6_header->ip6_nxt == IPPROTO_UDP) { + struct udphdr *udp_header = (struct udphdr *) (&ip6_header[1]); + fs_add_bool(fs, "success", 1); + fs_add_constchar(fs, "clas", "udp"); + fs_add_uint64(fs, "sport", ntohs(udp_header->uh_sport)); + fs_add_uint64(fs, "dport", ntohs(udp_header->uh_dport)); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_str"); + fs_add_uint64(fs, "udp_pkt_size", ntohs(udp_header->uh_ulen)); + // Verify that the UDP length is big enough for the header and + // at least one byte + uint16_t payload_len = + ntohs(udp_header->uh_ulen - sizeof(struct udphdr)); + if (payload_len > 0) { + uint32_t max_rlen = len - sizeof(struct ether_header) - + sizeof(struct ip6_hdr) - sizeof(struct udphdr); + uint32_t max_ilen = + ntohs(ip6_header->ip6_plen) - sizeof(struct udphdr); + + // Verify that the UDP length is inside of our received + // buffer + if (payload_len > max_rlen) { + payload_len = max_rlen; + } + // Verify that the UDP length is inside of our IPv6 packet + if (payload_len > max_ilen) { + payload_len = max_ilen; + } + fs_add_binary(fs, "data", payload_len, (void *) &udp_header[1], 0); + // Some devices reply with a zero UDP length but still + // return data, ignore the data + } else { + fs_add_null(fs, "data"); + } + } else if (ip6_header->ip6_nxt == IPPROTO_ICMPV6) { + struct icmp6_hdr *icmp6_header = (struct icmp6_hdr *) (&ip6_header[1]); + struct ip6_hdr *ip6_inner_header = (struct ip6_hdr *) &icmp6_header[1]; + struct udphdr * udp_inner_header = + (struct udphdr *) (&ip6_inner_header[1]); + // ICMPv6 unreach comes from another server (not the one we sent a + // probe to); But we will fix up saddr to be who we sent the + // probe to, in case you care. + fs_modify_string( + fs, "saddr", + make_ipv6_str((struct in6_addr *) &(ip6_inner_header->ip6_dst)), 1); + fs_add_bool(fs, "success", 0); + fs_add_string(fs, "clas", + (char *) get_icmp6_type_str(icmp6_header->icmp6_type), 0); + fs_add_uint64(fs, "sport", ntohs(udp_inner_header->uh_dport)); + fs_add_uint64(fs, "dport", ntohs(udp_inner_header->uh_sport)); + fs_add_string(fs, "icmp_responder", + make_ipv6_str((struct in6_addr *) &(ip6_header->ip6_src)), + 1); + fs_add_uint64(fs, "icmp_type", icmp6_header->icmp6_type); + fs_add_uint64(fs, "icmp_code", icmp6_header->icmp6_code); + fs_add_string(fs, "icmp_str", + (char *) get_icmp6_type_code_str( + icmp6_header->icmp6_type, icmp6_header->icmp6_code), + 0); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + } else { + fs_add_bool(fs, "success", 0); + fs_add_string(fs, "clas", (char *) "other", 0); + fs_add_null(fs, "sport"); + fs_add_null(fs, "dport"); + fs_add_null(fs, "icmp_responder"); + fs_add_null(fs, "icmp_type"); + fs_add_null(fs, "icmp_code"); + fs_add_null(fs, "icmp_str"); + fs_add_null(fs, "udp_pkt_size"); + fs_add_null(fs, "data"); + } +} + +// +// payload templete +int udp6_random_bytes(char *dst, int len, const unsigned char *charset, + int charset_len, aesrand_t *aes) { + int i; + for (i = 0; i < len; i++) { + *dst++ = charset[(aesrand_getword(aes) & 0xFFFFFFFF) % charset_len]; + } + + return i; +} + +int udp6_template_build(udp6_payload_template_t *t, char *out, unsigned int len, + struct ip6_hdr *ip6_hdr, struct udphdr *udp6_hdr, + aesrand_t *aes) { + udp6_payload_field_t *c; + char * p; + char * max; + char tmp[256]; + int full = 0; + unsigned int x, y; + uint8_t * u8; + uint16_t * u16; + + max = out + len; + p = out; + + for (x = 0; x < t->fcount; x++) { + c = t->fields[x]; + + // Exit the processing loop if our packet buffer would overflow + if (p + c->length >= max) { + full = 1; + return 0; + } + + switch (c->ftype) { + + // These fields have a specified output length value + + case UDP6_DATA: + if (!(c->data && c->length)) break; + memcpy(p, c->data, c->length); + p += c->length; + break; + + case UDP6_RAND_DIGIT: + p += udp6_random_bytes(p, c->length, charset_digit6, 10, aes); + break; + + case UDP6_RAND_ALPHA: + p += udp6_random_bytes(p, c->length, charset_alpha6, 52, aes); + break; + + case UDP6_RAND_ALPHANUM: + p += udp6_random_bytes(p, c->length, charset_alphanum6, 62, aes); + break; + + case UDP6_RAND_BYTE: + p += udp6_random_bytes(p, c->length, charset_all6, 256, aes); + break; + + // These fields need to calculate size on their own + + // TODO: Condense these case statements to remove redundant code + case UDP6_SADDR_A: + if (p + 39 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET6, (char *) &(ip6_hdr->ip6_src), tmp, + sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP6_DADDR_A: + if (p + 39 >= max) { + full = 1; + break; + } + // Write to stack and then memcpy in order to properly + // track length + inet_ntop(AF_INET6, (char *) &(ip6_hdr->ip6_dst), tmp, + sizeof(tmp) - 1); + memcpy(p, tmp, strlen(tmp)); + p += strlen(tmp); + break; + + case UDP6_SADDR_N: + if (p + 16 >= max) { + full = 1; + break; + } + + u8 = (uint8_t *) p; + uint8_t *ip6_src = (uint8_t *) &(ip6_hdr->ip6_src); + for (int i = 0; i < 16; i++) + u8[i] = ip6_src[i]; + p += 16; + break; + + case UDP6_DADDR_N: + if (p + 16 >= max) { + full = 1; + break; + } + + u8 = (uint8_t *) p; + uint8_t *ip6_dst = (uint8_t *) &(ip6_hdr->ip6_dst); + for (int i = 0; i < 16; i++) + u8[i] = ip6_dst[i]; + p += 16; + break; + + case UDP6_SPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *) p; + *u16 = udp6_hdr->uh_sport; + p += 2; + break; + + case UDP6_DPORT_N: + if (p + 2 >= max) { + full = 1; + break; + } + u16 = (uint16_t *) p; + *u16 = udp6_hdr->uh_sport; + p += 2; + break; + + case UDP6_SPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp6_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + + case UDP6_DPORT_A: + if (p + 5 >= max) { + full = 1; + break; + } + y = snprintf(tmp, 6, "%d", ntohs(udp6_hdr->uh_sport)); + memcpy(p, tmp, y); + p += y; + break; + } + + // Bail out if our packet buffer would overflow + if (full == 1) { + return 0; + } + } + + return p - out; +} + +// Free all buffers held by the payload template, including its own +void udp6_template_free(udp6_payload_template_t *t) { + for (unsigned int x = 0; x < t->fcount; x++) { + if (t->fields[x]->data) { + free(t->fields[x]->data); + t->fields[x]->data = NULL; + } + free(t->fields[x]); + t->fields[x] = NULL; + } + free(t->fields); + t->fields = NULL; + t->fcount = 0; + free(t); +} + +// Add a new field to the template +void udp6_template_add_field(udp6_payload_template_t * t, + udp6_payload_field_type_t ftype, + unsigned int length, char *data) { + udp6_payload_field_t *c; + + t->fcount++; + t->fields = xrealloc(t->fields, sizeof(udp6_payload_field_t) * t->fcount); + if (!t->fields) { + exit(1); + } + + t->fields[t->fcount - 1] = xmalloc(sizeof(udp6_payload_field_t)); + c = t->fields[t->fcount - 1]; + + if (!c) { + exit(1); + } + + c->ftype = ftype; + c->length = length; + c->data = data; +} + +// Convert a string field name to a field type, parsing any specified length +// value +int udp6_template_lookup_field(char *vname, udp6_payload_field_t *c) { + char * param; + unsigned int f; + unsigned int olen = 0; + unsigned int fcount = sizeof(udp6_payload_template_fields) / + sizeof(udp6_payload_template_fields[0]); + + param = strstr((const char *) vname, "="); + if (param) { + *param = '\0'; + param++; + } + + // Most field types treat their parameter as a generator output length + // unless it is ignored (ADDR, PORT, etc). + if (param) { + olen = atoi((const char *) param); + } + + // Find a field that matches the + for (f = 0; f < fcount; f++) { + + if (strcmp((char *) vname, udp6_payload_template_fields[f].name) == 0) { + c->ftype = udp6_payload_template_fields[f].ftype; + c->length = olen; + c->data = NULL; + return 1; + } + } + + // No match, skip and treat it as a data field + return 0; +} + +// Allocate a payload template and populate it by parsing a template file as a +// binary buffer +udp6_payload_template_t *udp6_template_load(char *buf, unsigned int len) { + udp6_payload_template_t *t = xmalloc(sizeof(udp6_payload_template_t)); + + // The last $ we encountered outside of a field specifier + char *dollar = NULL; + + // The last { we encountered outside of a field specifier + char *lbrack = NULL; + + // Track the start pointer of a data field (static) + char *s = buf; + + // Track the index into the template + char *p = buf; + + char * tmp; + unsigned int tlen; + + udp6_payload_field_t c; + + t->fcount = 0; + t->fields = NULL; + + while (p < (buf + len)) { + switch (*p) { + + case '$': + if ((dollar && !lbrack) || !dollar) { + dollar = p; + } + p++; + continue; + + case '{': + if (dollar && !lbrack) { + lbrack = p; + } + + p++; + continue; + + case '}': + if (!(dollar && lbrack)) { + p++; + continue; + } + + // Store the leading bytes before ${ as a data field + tlen = dollar - s; + if (tlen > 0) { + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp6_template_add_field(t, UDP6_DATA, tlen, tmp); + } + + tmp = xcalloc(1, p - lbrack); + memcpy(tmp, lbrack + 1, p - lbrack - 1); + + if (udp6_template_lookup_field(tmp, &c)) { + udp6_template_add_field(t, c.ftype, c.length, c.data); + + // Push the pointer past the } if this was a + // valid variable + s = p + 1; + } else { + + // Rewind back to the ${ sequence if this was an + // invalid variable + s = dollar; + } + + free(tmp); + break; + + default: + if (dollar && lbrack) { + p++; + continue; + } + } + + dollar = NULL; + lbrack = NULL; + + p++; + } + + // Store the trailing bytes as a final data field + if (s < p) { + tlen = p - s; + tmp = xmalloc(tlen); + memcpy(tmp, s, tlen); + udp6_template_add_field(t, UDP6_DATA, tlen, tmp); + } + + return t; +} + +static fielddef_t fields[] = { + {.name = "success", + .type = "bool", + .desc = "is response considered success"}, + {.name = "clas", + .type = "string", + .desc = "packet classification(type str):\n" + "\t\t\te.g., `udp', `unreach', `other'\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "sport", .type = "int", .desc = "UDP source port"}, + {.name = "dport", .type = "int", .desc = "UDP destination port"}, + {.name = "icmp_responder", + .type = "string", + .desc = "source IPv6 address of ICMPv6 message"}, + {.name = "icmp_type", .type = "int", .desc = "ICMPv6 message type"}, + {.name = "icmp_code", .type = "int", .desc = "ICMPv6 message code"}, + {.name = "icmp_str", + .type = "string", + .desc = "ICMPv6 message detail(code str):\n" + "\t\t\tuse `--probe-args=icmp-type-code-str' to list"}, + {.name = "udp_pkt_size", .type = "int", .desc = "UDP packet length"}, + {.name = "data", .type = "binary", .desc = "UDP payload"}}; + +probe_module_t module_udp6 = { + .ipv46_flag = 6, + .name = "udp", + .packet_length = 14 + 40 + 8 + 30, + .pcap_filter = "ip6 proto 17 || icmp6", + .pcap_snaplen = 1500, + .port_args = 1, + .global_init = &udp6_global_init, + .thread_init = &udp6_thread_init, + .make_packet = &udp6_make_packet, + .print_packet = &udp6_print_packet, + .validate_packet = &udp6_validate_packet, + .process_packet = &udp6_process_packet, + .close = &udp6_global_cleanup, + .fields = fields, + .numfields = sizeof(fields) / sizeof(fields[0]), + .helptext = + "Probe module that sends UDP packets to hosts.\n" + "Packets can optionally be templated based on destination host.\n" + " --probe-args=file:/path_to_packet_file\n" + " --probe-args=text:SomeText\n" + " --probe-args=hex:5061796c6f6164\n" + " --probe-args=dir:/dir_to_files\n" + " (each file named by port num, e.g., 53.pkt>txt>hex)\n" + " --probe-args=template:/path_to_template_file\n" + " --probe-args=template-fields (list allowed UDP template fields)\n" + " --probe-args=icmp-type-code-str (list allowed type/code str)", +}; diff --git a/src/probe_modules/module_udp6.h b/src/probe_modules/module_udp6.h new file mode 100644 index 0000000..8005383 --- /dev/null +++ b/src/probe_modules/module_udp6.h @@ -0,0 +1,104 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_MODULE_UDP6_H +#define XMAP_MODULE_UDP6_H + +#include +#include +#include +#include +#include + +#include "../../lib/includes.h" +#include "aesrand.h" +#include "state.h" +#include "types.h" + +typedef enum udp6_payload_field_type { + UDP6_DATA, + UDP6_SADDR_N, + UDP6_SADDR_A, + UDP6_DADDR_N, + UDP6_DADDR_A, + UDP6_SPORT_N, + UDP6_SPORT_A, + UDP6_DPORT_N, + UDP6_DPORT_A, + UDP6_RAND_BYTE, + UDP6_RAND_DIGIT, + UDP6_RAND_ALPHA, + UDP6_RAND_ALPHANUM +} udp6_payload_field_type_t; + +typedef struct udp6_payload_field_type_def { + const char * name; + const char * desc; + udp6_payload_field_type_t ftype; +} udp6_payload_field_type_def_t; + +typedef struct udp6_payload_field { + enum udp6_payload_field_type ftype; + unsigned int length; + char * data; +} udp6_payload_field_t; + +typedef struct udp6_payload_template { + unsigned int fcount; + struct udp6_payload_field **fields; +} udp6_payload_template_t; + +typedef struct udp6_payload_output { + int length; + char *data; +} udp6_payload_output_t; + +int udp6_global_init(struct state_conf *conf); + +int udp6_global_cleanup(UNUSED struct state_conf *xconf, + UNUSED struct state_send *xsend, + UNUSED struct state_recv *xrecv); + +int udp6_thread_init(void *buf, macaddr_t *src, macaddr_t *gw, void **arg_ptr); + +int udp6_make_packet(void *buf, UNUSED size_t *buf_len, ipaddr_n_t *src_ip, + ipaddr_n_t *dst_ip, port_h_t dst_port, uint8_t ttl, + int probe_num, UNUSED void *arg); + +void udp6_print_packet(FILE *fp, void *packet); + +int udp6_validate_packet(const struct ip *ip_hdr, uint32_t len, + UNUSED int *is_repeat); + +void udp6_process_packet(const u_char *packet, uint32_t len, fieldset_t *fs, + UNUSED struct timespec ts); + +// other +extern const char *udp6_unreach_strings[]; + +void udp6_set_num_sports(int x); + +// udp6 field +int udp6_template_build(udp6_payload_template_t *t, char *out, unsigned int len, + struct ip6_hdr *ip_hdr, struct udphdr *udp_hdr, + aesrand_t *aes); + +void udp6_template_free(udp6_payload_template_t *t); + +void udp6_template_add_field(udp6_payload_template_t * t, + udp6_payload_field_type_t ftype, + unsigned int length, char *data); + +int udp6_template_lookup_field(char *vname, udp6_payload_field_t *c); + +udp6_payload_template_t *udp6_template_load(char *buf, unsigned int len); + +#endif // XMAP_MODULE_UDP6_H diff --git a/src/probe_modules/packet.c b/src/probe_modules/packet.c new file mode 100644 index 0000000..2d8379c --- /dev/null +++ b/src/probe_modules/packet.c @@ -0,0 +1,209 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "stdio.h" +#include "string.h" + +#include "../../lib/includes.h" // should above packet.h +#include "../../lib/xalloc.h" + +#include "packet.h" + +#ifndef NDEBUG + +void print_macaddr(struct ifreq *i) { + printf("Device %s -> Ethernet %02x:%02x:%02x:%02x:%02x:%02x\n", i->ifr_name, + (int) ((unsigned char *) &i->ifr_addr.sa_data)[0], + (int) ((unsigned char *) &i->ifr_addr.sa_data)[1], + (int) ((unsigned char *) &i->ifr_addr.sa_data)[2], + (int) ((unsigned char *) &i->ifr_addr.sa_data)[3], + (int) ((unsigned char *) &i->ifr_addr.sa_data)[4], + (int) ((unsigned char *) &i->ifr_addr.sa_data)[5]); +} + +#endif /* NDEBUG */ + +#define IP_ADDR_LEN_STR 20 + +void fprintf_ip_header(FILE *fp, struct ip *iph) { + struct in_addr *s = (struct in_addr *) &(iph->ip_src); + struct in_addr *d = (struct in_addr *) &(iph->ip_dst); + + char srcip[IP_ADDR_LEN_STR + 1]; + char dstip[IP_ADDR_LEN_STR + 1]; + // inet_ntoa is a const char * so we if just call it in + // fprintf, you'll get back wrong results since we're + // calling it twice. + strncpy(srcip, inet_ntoa(*s), IP_ADDR_LEN_STR - 1); + strncpy(dstip, inet_ntoa(*d), IP_ADDR_LEN_STR - 1); + + srcip[IP_ADDR_LEN_STR] = '\0'; + dstip[IP_ADDR_LEN_STR] = '\0'; + + fprintf(fp, + "IPv4\n" + "\tVersion(4b)\t\t: 4\n" + "\tHeader Length(4b)\t: %d\n" + "\tService Field(1B)\t: 0x%02x\n" + "\tTotal Length(2B)\t: %d\n" + "\tIdentification(2B)\t: %d\n" + "\tFlag(2B)\t\t: 0x%04x\n" + "\tTime to live(1B)\t: %d\n" + "\tProtocol(1B)\t\t: %d\n" + "\tChecksum(2B)\t\t: 0x%04x\n" + "\tSource(4B)\t\t: %s\n" + "\tDestination(4B)\t\t: %s\n", + iph->ip_hl * 4, iph->ip_tos, ntohs(iph->ip_len), ntohs(iph->ip_id), + ((uint16_t *) iph)[3], iph->ip_ttl, iph->ip_p, iph->ip_sum, srcip, + dstip); +} + +void fprintf_ip6_header(FILE *fp, struct ip6_hdr *ip6h) { + struct in6_addr *s = (struct in6_addr *) &(ip6h->ip6_src); + struct in6_addr *d = (struct in6_addr *) &(ip6h->ip6_dst); + + char srcip[INET6_ADDRSTRLEN + 1]; + char dstip[INET6_ADDRSTRLEN + 1]; + + // TODO: Is restrict correct here? + inet_ntop(AF_INET6, s, (char *restrict) &srcip, INET6_ADDRSTRLEN); + inet_ntop(AF_INET6, d, (char *restrict) &dstip, INET6_ADDRSTRLEN); + + srcip[INET6_ADDRSTRLEN] = '\0'; + dstip[INET6_ADDRSTRLEN] = '\0'; + + fprintf(fp, + "IPv6\n" + "\tVersion(4b)\t\t: 6\n" + "\tTraffic Class(8b)\t: 0x%x%x\n" + "\tFlow Label(20b)\t\t: 0x%x%02x%02x\n" + "\tPayload Length(2B)\t: %d\n" + "\tNext Header(1B)\t\t: %d\n" + "\tHop Limit(1B)\t\t: %u\n" + "\tSource(16B)\t\t: %s\n" + "\tDestination(16B)\t: %s\n", + ((uint8_t *) ip6h)[0] & 0x0fu, ((uint8_t *) ip6h)[1] & 0xf0u, + ((uint8_t *) ip6h)[1] & 0x0fu, ((uint8_t *) ip6h)[2], + ((uint8_t *) ip6h)[3], ntohs(ip6h->ip6_plen), ip6h->ip6_nxt, + ip6h->ip6_hlim, srcip, dstip); +} + +void fprintf_eth_header(FILE *fp, struct ether_header *ethh) { + if (!xconf.send_ip_pkts) { + fprintf(fp, + "Ethernet\n" + "\tDestination(6B)\t\t: %02x:%02x:%02x:%02x:%02x:%02x\n" + "\tSource(6B)\t\t: %02x:%02x:%02x:%02x:%02x:%02x\n" + "\tType(2B)\t\t: 0x%04x\n", + (int) ((unsigned char *) ethh->ether_dhost)[0], + (int) ((unsigned char *) ethh->ether_dhost)[1], + (int) ((unsigned char *) ethh->ether_dhost)[2], + (int) ((unsigned char *) ethh->ether_dhost)[3], + (int) ((unsigned char *) ethh->ether_dhost)[4], + (int) ((unsigned char *) ethh->ether_dhost)[5], + (int) ((unsigned char *) ethh->ether_shost)[0], + (int) ((unsigned char *) ethh->ether_shost)[1], + (int) ((unsigned char *) ethh->ether_shost)[2], + (int) ((unsigned char *) ethh->ether_shost)[3], + (int) ((unsigned char *) ethh->ether_shost)[4], + (int) ((unsigned char *) ethh->ether_shost)[5], + ntohs(ethh->ether_type)); + } +} + +void make_eth_header(struct ether_header *ethh, macaddr_t *src, + macaddr_t *dst) { + // Create a frame with IPv4 ethertype by default + make_eth_header_ethertype(ethh, src, dst, ETHERTYPE_IP); +} + +void make_eth6_header(struct ether_header *ethh, macaddr_t *src, + macaddr_t *dst) { + // Create a frame with IPv6 ethertype by default + make_eth_header_ethertype(ethh, src, dst, ETHERTYPE_IPV6); +} + +void make_eth_header_ethertype(struct ether_header *ethh, macaddr_t *src, + macaddr_t *dst, uint16_t ethertype) { + memcpy(ethh->ether_shost, src, ETHER_ADDR_LEN); + memcpy(ethh->ether_dhost, dst, ETHER_ADDR_LEN); + ethh->ether_type = htons(ethertype); +} + +void make_ip_header(struct ip *iph, uint8_t protocol, uint16_t ip_len) { + iph->ip_hl = 5; // Internet Header Length + iph->ip_v = 4; // IPv4 + iph->ip_tos = 0; // Type of Service + iph->ip_len = htons(ip_len); + iph->ip_id = htons(54321); // identification number + iph->ip_off = 0; // fragmentation flag + iph->ip_ttl = MAXTTL; // time to live (TTL) + iph->ip_p = protocol; // upper layer protocol => TCP + // we set the checksum = 0 for now because that's + // what it needs to be when we run the IP checksum + iph->ip_sum = 0; +} + +void make_ip6_header(struct ip6_hdr *iph, uint8_t protocol, uint16_t pl_len) { + iph->ip6_ctlun.ip6_un2_vfc = 0x60; // 4 bits version, top 4 bits class + iph->ip6_plen = htons(pl_len); // payload length + iph->ip6_nxt = protocol; // next header + iph->ip6_hlim = MAXTTL; // hop limit +} + +void make_icmp6_header(struct icmp6_hdr *buf) { + buf->icmp6_type = ICMP6_ECHO_REQUEST; + buf->icmp6_code = 0; + buf->icmp6_cksum = 0; +} + +void make_icmp_header(struct icmp *buf) { + buf->icmp_type = ICMP_ECHO; + buf->icmp_code = 0; + buf->icmp_cksum = 0; +} + +void make_tcp_header(struct tcphdr *tcp_header, uint16_t th_flags) { + tcp_header->th_seq = random(); + tcp_header->th_ack = 0; + tcp_header->th_x2 = 0; + tcp_header->th_off = 5; // data offset + tcp_header->th_flags = 0; + tcp_header->th_flags |= th_flags; + tcp_header->th_win = htons(65535); // largest possible window + tcp_header->th_sum = 0; + tcp_header->th_urp = 0; + // tcp_header->th_dport +} + +void make_udp_header(struct udphdr *udp_header, uint16_t len) { + // udp_header->uh_sport + // udp_header->uh_dport + udp_header->uh_ulen = htons(len); + udp_header->uh_sum = 0; // checksum ignored in IPv4 if 0 +} + +// Note: caller must free return value +char *make_ip_str(uint32_t ip) { + struct in_addr t; + t.s_addr = ip; + const char *temp = inet_ntoa(t); + char * retv = xmalloc(strlen(temp) + 1); + strcpy(retv, temp); + return retv; +} + +// Note: caller must free return value +char *make_ipv6_str(struct in6_addr *ipv6) { + char *retv = xmalloc(INET6_ADDRSTRLEN + 1); + inet_ntop(AF_INET6, ipv6, retv, INET6_ADDRSTRLEN); + return retv; +} diff --git a/src/probe_modules/packet.h b/src/probe_modules/packet.h new file mode 100644 index 0000000..3a81d9b --- /dev/null +++ b/src/probe_modules/packet.h @@ -0,0 +1,354 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_PACKET_H +#define XMAP_PACKET_H + +#include "../../lib/includes.h" +#include "../state.h" + +#define MAX_PACKET_SIZE 4096 + +typedef unsigned short __attribute__((__may_alias__)) alias_unsigned_short; + +void make_eth_header(struct ether_header *ethh, macaddr_t *src, macaddr_t *dst); + +void make_eth6_header(struct ether_header *ethh, macaddr_t *src, + macaddr_t *dst); + +void make_eth_header_ethertype(struct ether_header *ethh, macaddr_t *src, + macaddr_t *dst, uint16_t ether_type); + +void make_ip_header(struct ip *iph, uint8_t protocol, uint16_t ip_len); + +void make_ip6_header(struct ip6_hdr *iph, uint8_t protocol, uint16_t pl_len); + +void make_tcp_header(struct tcphdr *tcp_header, uint16_t len); + +void make_icmp_header(struct icmp *); + +void make_icmp6_header(struct icmp6_hdr *); + +void make_udp_header(struct udphdr *udp_header, uint16_t len); + +void fprintf_ip_header(FILE *fp, struct ip *iph); + +void fprintf_ip6_header(FILE *fp, struct ip6_hdr *ip6h); + +void fprintf_eth_header(FILE *fp, struct ether_header *ethh); + +static inline unsigned short in_checksum(unsigned short *ip_pkt, int len) { + unsigned long sum = 0; + for (int nwords = len / 2; nwords > 0; nwords--) { + sum += *ip_pkt++; + } + sum = (sum >> 16) + (sum & 0xffff); + sum += (sum >> 16); + + return (unsigned short) (~sum); +} + +__attribute__((unused)) static inline unsigned short + ip_checksum_(unsigned short *buf) { + return in_checksum(buf, (int) sizeof(struct ip)); +} + +__attribute__((unused)) static inline unsigned short + icmp_checksum_(unsigned short *buf) { + return in_checksum(buf, (int) sizeof(struct icmp)); +} + +__attribute__((unused)) static inline uint16_t + icmp_checksum(struct icmp *icmp_pkt, size_t data_len) { + unsigned short icmp_len = sizeof(struct icmp) + data_len; + unsigned long sum = 0; + int nleft = icmp_len; + unsigned short *w = (unsigned short *) icmp_pkt; + + // calculate the checksum for the icmp header and icmp data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + // if nleft is 1 there ist still on byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & 0x00FF; + } + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + + // Take the one's complement of sum + return (unsigned short) (~sum); +} + +__attribute__((unused)) static inline uint16_t + icmp6_checksum(struct in6_addr *saddr, struct in6_addr *daddr, + struct icmp6_hdr *icmp6_pkt, size_t data_len) { + alias_unsigned_short *src_addr = (alias_unsigned_short *) saddr; + alias_unsigned_short *dest_addr = (alias_unsigned_short *) daddr; + unsigned short icmp6_len = sizeof(struct icmp6_hdr) + data_len; + unsigned long sum = 0; + int nleft = icmp6_len; + unsigned short * w = (unsigned short *) icmp6_pkt; + + // calculate the checksum for the icmpv6 header and icmpv6 data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + // if nleft is 1 there ist still on byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & 0x00FF; + } + + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += src_addr[2]; + sum += src_addr[3]; + sum += src_addr[4]; + sum += src_addr[5]; + sum += src_addr[6]; + sum += src_addr[7]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += dest_addr[2]; + sum += dest_addr[3]; + sum += dest_addr[4]; + sum += dest_addr[5]; + sum += dest_addr[6]; + sum += dest_addr[7]; + sum += htons(icmp6_len); + sum += htons(IPPROTO_ICMPV6); + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + + // Take the one's complement of sum + return (unsigned short) (~sum); +} + +__attribute__((unused)) static inline uint16_t + udp6_checksum(struct in6_addr *saddr, struct in6_addr *daddr, + struct udphdr *udp_pkt) { + alias_unsigned_short *src_addr = (alias_unsigned_short *) saddr; + alias_unsigned_short *dest_addr = (alias_unsigned_short *) daddr; + unsigned long sum = 0; + int nleft = ntohs(udp_pkt->uh_ulen); + unsigned short * w = (unsigned short *) udp_pkt; + + // calculate the checksum for the udp6 header and udp6 data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + // if nleft is 1 there ist still on byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & 0x00FF; + } + + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += src_addr[2]; + sum += src_addr[3]; + sum += src_addr[4]; + sum += src_addr[5]; + sum += src_addr[6]; + sum += src_addr[7]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += dest_addr[2]; + sum += dest_addr[3]; + sum += dest_addr[4]; + sum += dest_addr[5]; + sum += dest_addr[6]; + sum += dest_addr[7]; + sum += udp_pkt->uh_ulen; // net order + sum += htons(IPPROTO_UDP); + + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + + // Take the one's complement of sum + return (unsigned short) (~sum); +} + +__attribute__((unused)) static inline uint16_t + udp_checksum(uint32_t saddr, uint32_t daddr, struct udphdr *udp_pkt) { + alias_unsigned_short *src_addr = (alias_unsigned_short *) &saddr; + alias_unsigned_short *dest_addr = (alias_unsigned_short *) &daddr; + unsigned long sum = 0; + int nleft = ntohs(udp_pkt->uh_ulen); + unsigned short * w = (unsigned short *) udp_pkt; + + // calculate the checksum for the udp header and udp data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + // if nleft is 1 there ist still on byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & 0x00FF; + } + + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += udp_pkt->uh_ulen; // net order + sum += htons(IPPROTO_UDP); + + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + + // Take the one's complement of sum + return (unsigned short) (~sum); +} + +__attribute__((unused)) static inline uint16_t + tcp6_checksum(struct in6_addr *saddr, struct in6_addr *daddr, + struct tcphdr *tcp_pkt, unsigned short len_tcp) { + uint16_t *src_addr = (uint16_t *) saddr; + uint16_t *dest_addr = (uint16_t *) daddr; + + unsigned long sum = 0; + int nleft = len_tcp; + unsigned short *w = (unsigned short *) tcp_pkt; + + // calculate the checksum for the tcp6 header and tcp6 data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + // if nleft is 1 there ist still on byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & 0x00FF; + } + + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += src_addr[2]; + sum += src_addr[3]; + sum += src_addr[4]; + sum += src_addr[5]; + sum += src_addr[6]; + sum += src_addr[7]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += dest_addr[2]; + sum += dest_addr[3]; + sum += dest_addr[4]; + sum += dest_addr[5]; + sum += dest_addr[6]; + sum += dest_addr[7]; + sum += htons(len_tcp); + sum += htons(IPPROTO_TCP); + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + + // Take the one's complement of sum + return (unsigned short) (~sum); +} + +__attribute__((unused)) static inline uint16_t + tcp_checksum(unsigned short len_tcp, uint32_t saddr, uint32_t daddr, + struct tcphdr *tcp_pkt) { + alias_unsigned_short *src_addr = (alias_unsigned_short *) &saddr; + alias_unsigned_short *dest_addr = (alias_unsigned_short *) &daddr; + + unsigned long sum = 0; + int nleft = len_tcp; + unsigned short *w = (unsigned short *) tcp_pkt; + // calculate the checksum for the tcp header and tcp data + while (nleft > 1) { + sum += *w++; + nleft -= 2; + } + + // if nleft is 1 there ist still on byte left. + // We add a padding byte (0xFF) to build a 16bit word + if (nleft > 0) { + sum += *w & 0x00FF; + } + + // add the pseudo header + sum += src_addr[0]; + sum += src_addr[1]; + sum += dest_addr[0]; + sum += dest_addr[1]; + sum += htons(len_tcp); + sum += htons(IPPROTO_TCP); + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + + // Take the one's complement of sum + return (unsigned short) (~sum); +} + +// Returns 0 if dst_port is outside the expected valid range, non-zero otherwise +__attribute__((unused)) static inline int + check_src_port(uint16_t sport, int num_ports, uint8_t *validation) { + if (sport > xconf.source_port_last || sport < xconf.source_port_first) { + return 0; + } + + int32_t to_validate = sport - xconf.source_port_first; + int32_t min = + ((((uint16_t) validation[0]) << 8u) + validation[1]) % num_ports; + int32_t max = ((((uint16_t) validation[0]) << 8u) + validation[1] + + xconf.packet_streams - 1) % + num_ports; + + return (((max - min) % num_ports) >= ((to_validate - min) % num_ports)); +} + +__attribute__((unused)) static inline uint16_t + get_src_port(int num_ports, int probe_num, uint8_t *validation) { + return xconf.source_port_first + + (((((uint16_t) validation[0]) << 8u) + validation[1] + probe_num) % + num_ports); +} + +__attribute__((unused)) static inline uint16_t + icmp_get_idnum(uint8_t *validation) { + return (((uint16_t) validation[0]) << 8u) + validation[1]; +} + +__attribute__((unused)) static inline uint16_t + icmp_get_seqnum(uint8_t *validation) { + return (((uint16_t) validation[2]) << 8u) + validation[3]; +} + +__attribute__((unused)) static inline uint32_t + tcp_get_seqnum(uint8_t *validation) { + return (((uint32_t) validation[0]) << 24u) + + (((uint32_t) validation[1]) << 16u) + + (((uint32_t) validation[2]) << 8u) + (((uint32_t) validation[3])); +} + +// Note: caller must free return value +char *make_ip_str(uint32_t ip); + +char *make_ipv6_str(struct in6_addr *ipv6); + +#endif // XMAP_PACKET_H diff --git a/src/probe_modules/packet_icmp.c b/src/probe_modules/packet_icmp.c new file mode 100644 index 0000000..23f0fc5 --- /dev/null +++ b/src/probe_modules/packet_icmp.c @@ -0,0 +1,220 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "packet_icmp.h" + +#include "../../lib/lockfd.h" + +#define ICMP_TYPE_MAX_INDEX 40 +#define ICMP_CODE_MAX_INDEX 16 + +static const char *ICMP_TYPE_STR[] = { + "echoreply", + "other", + "other", + "unreach", + "srcquench", // 0 - 4 + "redirect", + "althostaddr", + "other", + "echo", + "rtradvert", // 5 - 9 + "rtrsolicit", + "timexceed", + "paramprob", + "tstamp", + "tstampreply", // 10 - 14 + "info", + "inforeply", + "mask", + "maskreply", + "other", // 15 - 19 + "other", + "other", + "other", + "other", + "other", // 20 - 24 + "other", + "other", + "other", + "other", + "other", // 25 - 29 + "traceroute", + "dataconverr", + "mobile-redirect", + "ipv6-whereareyou", + "ipv6-iamhere", // 30 - 34 + "mobile-reg", + "mobile-regreply", + "dns", + "dnsreply", + "skip", // 35 - 39 + "photuris", // 40 +}; + +static int ICMP_TYPE_CODE_MAX_INDEX[] = { + 0, 0, 0, 15, 0, // 0 - 4 + 3, 0, 0, 0, 16, // 5 - 9 + 0, 1, 2, 0, 0, // 10 - 14 + 0, 0, 0, 0, 0, // 15 - 19 + 0, 0, 0, 0, 0, // 20 - 24 + 0, 0, 0, 0, 0, // 25 - 29 + 0, 0, 0, 0, 0, // 30 - 34 + 0, 0, 0, 0, 0, // 35 - 39 + 5 // 40 +}; + +static const char *ICMP_TYPE_CODE_STR[ICMP_TYPE_MAX_INDEX + + 1][ICMP_CODE_MAX_INDEX + 1] = { + { + // echoreply: 0 + "no code", + }, + {// other: 1 + "unknown"}, + {// other: 2 + "unknown"}, + { + // unreach: 3 + "net unreachable", "host unreachable", "protocol unreachable", // 0 - 2 + "port unreachable", "fragments required", + "source route failed", // 3 - 5 + "dst net unknown", "dst host unknown", "src host isolated", // 6 - 8 + "dst net admin prohibited", "dst host admin prohibited", + "dst net unreachable tos", // 9 - 11 + "dst host unreachable tos", "communication admin prohibited", + "host precedence violation", // 12 - 14 + "precedence cutoff", // 15 + }, + { + // echoreply: 4 + "no code", + }, + { + // redirect: 5 + "net redirect", "host redirect", "net redirect tos", + "host redirect tos", // 0 - 3 + }, + { + // althostaddr: 6 + "alternate address", + }, + {// other: 7 + "unknown"}, + {// echo: 8 + "no code"}, + { + // rtradvert: 9 + "router advertisement normal", "unknown", "unknown", "unknown", // 0 - 3 + "unknown", "unknown", "unknown", "unknown", // 4 - 7 + "unknown", "unknown", "unknown", "unknown", // 8 - 11 + "unknown", "unknown", "unknown", "unknown", // 12 - 15 + "not route common traffic", // 16 + }, + {// rtrsolicit: 10 + "no code"}, + { + // timexceed: 11 + "transit time exceeded", + "fragment reassembly time exceeded", + }, + { + // paramprob: 12 + "error at pointer", + "option required", + "bad length", + }, + {// tstamp: 13 + "no code"}, + {// tstampreply: 14 + "no code"}, + {// info: 15 + "no code"}, + {// inforeply: 16 + "no code"}, + {// mask: 17 + "no code"}, + {// maskreply: 18 + "no code"}, + {// other: 19 + "unknown"}, + {// other: 20 + "unknown"}, + {// other: 21 + "unknown"}, + {// other: 22 + "unknown"}, + {// other: 23 + "unknown"}, + {// other: 24 + "unknown"}, + {// other: 25 + "unknown"}, + {// other: 26 + "unknown"}, + {// other: 27 + "unknown"}, + {// other: 28 + "unknown"}, + {// other: 29 + "unknown"}, + {// traceroute: 30 + "no code"}, + {// dataconverr: 31 + "no code"}, + {// mobile-redirect: 32 + "no code"}, + {// ipv6-whereareyou: 33 + "no code"}, + {// ipv6-iamhere: 34 + "no code"}, + {// mobile-reg: 35 + "no code"}, + {// mobile-regreply: 36 + "no code"}, + {// dns: 37 + "no code"}, + {// dnsreply: 38 + "no code"}, + {// skip: 39 + "no code"}, + {// photuris: 40 + "bad spi", "authentication failed", "decompression failed", + "decryption failed", "need authentication", "need authorization"}, +}; + +const char *get_icmp_type_str(int type) { + if (type > ICMP_TYPE_MAX_INDEX) return "other"; + + return ICMP_TYPE_STR[type]; +} + +const char *get_icmp_type_code_str(int type, int code) { + if (type > ICMP_TYPE_MAX_INDEX) return "unknown"; + if (code > ICMP_TYPE_CODE_MAX_INDEX[type]) return "unknown"; + + return ICMP_TYPE_CODE_STR[type][code]; +} + +void print_icmp_type_code_str() { + lock_file(stderr); + fprintf(stderr, "------------------------------------------------------\n"); + for (int i = 0; i <= ICMP_TYPE_MAX_INDEX; i++) { + fprintf(stderr, "%-25s %s\n", "icmp-type-str(code)", + "icmp-code-str(code)"); + fprintf(stderr, + "------------------------------------------------------\n"); + fprintf(stderr, "%s(%d)\n", ICMP_TYPE_STR[i], i); + for (int j = 0; j <= ICMP_TYPE_CODE_MAX_INDEX[i]; j++) + fprintf(stderr, "%-25s %s(%d)\n", "", ICMP_TYPE_CODE_STR[i][j], j); + fprintf(stderr, + "------------------------------------------------------\n"); + } + unlock_file(stderr); +} diff --git a/src/probe_modules/packet_icmp.h b/src/probe_modules/packet_icmp.h new file mode 100644 index 0000000..4c8fa78 --- /dev/null +++ b/src/probe_modules/packet_icmp.h @@ -0,0 +1,19 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_PACKET_ICMP_H +#define XMAP_PACKET_ICMP_H + +const char *get_icmp_type_str(int type); + +const char *get_icmp_type_code_str(int type, int code); + +void print_icmp_type_code_str(); + +#endif // XMAP_PACKET_ICMP_H diff --git a/src/probe_modules/packet_icmp6.c b/src/probe_modules/packet_icmp6.c new file mode 100644 index 0000000..88c226f --- /dev/null +++ b/src/probe_modules/packet_icmp6.c @@ -0,0 +1,89 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "packet_icmp6.h" + +#include "../../lib/lockfd.h" + +#define ICMP6_TYPE_MAX_INDEX 4 +#define ICMP6_CODE_MAX_INDEX 10 + +static const char *ICMP6_TYPE_STR[] = { + "other", "unreach", "packetobig", "timexceed", "paramprob", // 1 - 4 +}; + +static int ICMP6_TYPE_CODE_MAX_INDEX[] = { + 0, 8, 0, 1, 10, // 1 - 4 +}; + +static const char + *ICMP6_TYPE_CODE_STR[ICMP6_TYPE_MAX_INDEX + 1][ICMP6_CODE_MAX_INDEX + 1] = { + { + // other: 0 + "unknown", + }, + {// unreach: 1 + "no route to dst", "communication admin prohibited", + "beyond src addr scope", "addr unreach", "port unreach", + "src addr policy failed", "route rejected", "src routing header error", + "too long headers"}, + { + // packetobig: 2 + "no code", + }, + { + // timexceed: 3 + "transit time exceeded", + "fragment reassembly time exceeded", + }, + { + // paramprob: 4 + "header field error", + "unknown next header", + "unknown ipv6 option", + "incomplete ipv6 header chain in 1st fragement", + "sr upper-layer header error", + "unknown next header at intermediate node", + "header-ext too big", + "header-ext chain too big", + "header-ext too many", + "header-ext option too many", + "option goo big", + }, +}; + +const char *get_icmp6_type_str(int type) { + if (type > ICMP6_TYPE_MAX_INDEX) return "other"; + + return ICMP6_TYPE_STR[type]; +} + +const char *get_icmp6_type_code_str(int type, int code) { + if (type > ICMP6_TYPE_MAX_INDEX) return "unknown"; + if (code > ICMP6_TYPE_CODE_MAX_INDEX[type]) return "unknown"; + + return ICMP6_TYPE_CODE_STR[type][code]; +} + +void print_icmp6_type_code_str() { + lock_file(stderr); + fprintf(stderr, "------------------------------------------------------\n"); + for (int i = 0; i <= ICMP6_TYPE_MAX_INDEX; i++) { + fprintf(stderr, "%-25s %s\n", "icmpv6-type-str(code)", + "icmpv6-code-str(code)"); + fprintf(stderr, + "------------------------------------------------------\n"); + fprintf(stderr, "%s(%d)\n", ICMP6_TYPE_STR[i], i); + for (int j = 0; j <= ICMP6_TYPE_CODE_MAX_INDEX[i]; j++) + fprintf(stderr, "%-25s %s(%d)\n", "", ICMP6_TYPE_CODE_STR[i][j], j); + fprintf(stderr, + "------------------------------------------------------\n"); + } + unlock_file(stderr); +} diff --git a/src/probe_modules/packet_icmp6.h b/src/probe_modules/packet_icmp6.h new file mode 100644 index 0000000..d24d55f --- /dev/null +++ b/src/probe_modules/packet_icmp6.h @@ -0,0 +1,19 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_PACKET_ICMP6_H +#define XMAP_PACKET_ICMP6_H + +const char *get_icmp6_type_str(int type); + +const char *get_icmp6_type_code_str(int type, int code); + +void print_icmp6_type_code_str(); + +#endif // XMAP_PACKET_ICMP6_H diff --git a/src/probe_modules/probe_modules.c b/src/probe_modules/probe_modules.c new file mode 100644 index 0000000..2d727a5 --- /dev/null +++ b/src/probe_modules/probe_modules.c @@ -0,0 +1,144 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "probe_modules.h" + +#include +#include +#include +#include + +#include "packet.h" + +#include "../../lib/includes.h" +#include "../../lib/xalloc.h" +#include "../fieldset.h" + +extern probe_module_t module_icmp_echo; +extern probe_module_t module_udp; +extern probe_module_t module_tcp_syn; +extern probe_module_t module_icmp6_echo; +extern probe_module_t module_udp6; +extern probe_module_t module_tcp6_syn; +extern probe_module_t module_icmp6_echo_tmxd; +extern probe_module_t module_icmp6_echo_gw; +// ADD YOUR MODULE HERE + +probe_module_t *probe_modules[] = { + &module_udp, &module_tcp_syn, &module_icmp_echo, + + &module_udp6, &module_tcp6_syn, &module_icmp6_echo, + &module_icmp6_echo_gw, &module_icmp6_echo_tmxd, + // ADD YOUR MODULE HERE +}; + +probe_module_t *get_probe_module_by_name(const char *name, int ipv46_flag) { + int len = (int) (sizeof(probe_modules) / sizeof(probe_modules[0])); + for (int i = 0; i < len; i++) { + if (probe_modules[i]->ipv46_flag != ipv46_flag) continue; + if (!strcmp(probe_modules[i]->name, name)) { + return probe_modules[i]; + } + } + + return NULL; +} + +void print_probe_modules(int ipv46_flag) { + int len = (int) (sizeof(probe_modules) / sizeof(probe_modules[0])); + for (int i = 0; i < len; i++) { + if (probe_modules[i]->ipv46_flag != ipv46_flag) continue; + printf("%s\n", probe_modules[i]->name); + } +} + +void fs_add_ip_fields(fieldset_t *fs, struct ip *ip) { + // WARNING: you must update fs_ip_fields_len as well + // as the definitions set (ip_fiels) if you + // change the fields added below: + fs_add_string(fs, "saddr", make_ip_str(ip->ip_src.s_addr), 1); + fs_add_string(fs, "daddr", make_ip_str(ip->ip_dst.s_addr), 1); + fs_add_uint64(fs, "ipid", ntohs(ip->ip_id)); + fs_add_uint64(fs, "ttl", ip->ip_ttl); +} + +void fs_add_ipv6_fields(fieldset_t *fs, struct ip6_hdr *ipv6_hdr) { + // WARNING: you must update fs_ip_fields_len as well + // as the definitions set (ip6_fiels) if you + // change the fields added below: + fs_add_string(fs, "saddr", + make_ipv6_str((struct in6_addr *) &(ipv6_hdr->ip6_src)), 1); + fs_add_string(fs, "daddr", + make_ipv6_str((struct in6_addr *) &(ipv6_hdr->ip6_dst)), 1); + fs_add_uint64(fs, "hlim", ipv6_hdr->ip6_ctlun.ip6_un1.ip6_un1_hlim); +} + +#define TIMESTR_LEN 55 + +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown) { + fs_add_bool(fs, "repeat", is_repeat); + fs_add_bool(fs, "cooldown", in_cooldown); + + char * timestr = xmalloc(TIMESTR_LEN + 1); + char * timestr_ms = xmalloc(TIMESTR_LEN + 1); + struct timeval t; + gettimeofday(&t, NULL); + struct tm *ptm = localtime(&t.tv_sec); + strftime(timestr, TIMESTR_LEN, "%Y-%m-%dT%H:%M:%S.%%03d%z", ptm); + snprintf(timestr_ms, TIMESTR_LEN, timestr, t.tv_usec / 1000); + free(timestr); + fs_add_string(fs, "timestamp_str", timestr_ms, 1); + fs_add_uint64(fs, "timestamp_ts", (uint64_t) t.tv_sec); + fs_add_uint64(fs, "timestamp_us", (uint64_t) t.tv_usec); +} + +int ip_fields_len = 4; +fielddef_t ip_fields[] = { + {.name = "saddr", + .type = "string", + .desc = "source IP address of response"}, + {.name = "daddr", + .type = "string", + .desc = "destination IP address of response"}, + {.name = "ipid", + .type = "int", + .desc = "IP identification number of response"}, + {.name = "ttl", .type = "int", .desc = "time-to-live of response packet"}}; + +int ip6_fields_len = 3; +fielddef_t ip6_fields[] = { + {.name = "saddr", + .type = "string", + .desc = "source IPv6 address of response"}, + {.name = "daddr", + .type = "string", + .desc = "destination IPv6 address of response"}, + {.name = "hlim", .type = "int", .desc = "hop-limit of response packet"}, +}; + +int sys_fields_len = 5; +fielddef_t sys_fields[] = { + {.name = "repeat", + .type = "bool", + .desc = "is response a repeat response from host"}, + {.name = "cooldown", + .type = "bool", + .desc = "was response received during the cooldown period"}, + {.name = "timestamp_str", + .type = "string", + .desc = "timestamp of when response arrived in ISO8601 format."}, + {.name = "timestamp_ts", + .type = "int", + .desc = "timestamp of when response arrived in seconds since Epoch"}, + {.name = "timestamp_us", + .type = "int", + .desc = "microsecond part of timestamp (e.g. microseconds since " + "`timestamp-ts')"}}; diff --git a/src/probe_modules/probe_modules.h b/src/probe_modules/probe_modules.h new file mode 100644 index 0000000..4a84adf --- /dev/null +++ b/src/probe_modules/probe_modules.h @@ -0,0 +1,93 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_PROBE_MODULES_H +#define XMAP_PROBE_MODULES_H + +#include "../fieldset.h" +#include "../state.h" + +#define OUTPUT_TYPE_STATIC 1 +#define OUTPUT_TYPE_DYNAMIC 2 + +#define PACKET_VALID 1 +#define PACKET_INVALID 0 + +typedef struct probe_response_type { + const uint8_t is_success; + const char * name; +} response_type_t; + +typedef int (*probe_global_init_cb)(struct state_conf *); + +typedef int (*probe_thread_init_cb)(void *packet_buf, macaddr_t *src_mac, + macaddr_t *gw_mac, void **arg_ptr); + +typedef int (*probe_make_packet_cb)(void *packet_buf, size_t *buf_len, + ipaddr_n_t *src_ip, ipaddr_n_t *dst_ip, + port_h_t dst_port, uint8_t ttl, + int probe_num, void *arg); + +typedef void (*probe_print_packet_cb)(FILE *, void *packet_buf); + +typedef int (*probe_validate_packet_cb)(const struct ip *ip_hdr, uint32_t len, + int *is_repeat); + +typedef void (*probe_classify_packet_cb)(const u_char *packet_buf, uint32_t len, + fieldset_t * fs, + const struct timespec ts); + +typedef int (*probe_close_cb)(struct state_conf *, struct state_send *, + struct state_recv *); + +typedef struct probe_module { + int ipv46_flag; + const char *name; + size_t packet_length; + const char *pcap_filter; + size_t pcap_snaplen; + + // Should XMap complain if the user hasn't specified valid + // source and target port numbers? + uint8_t port_args; + + probe_global_init_cb global_init; + probe_thread_init_cb thread_init; + probe_make_packet_cb make_packet; + probe_print_packet_cb print_packet; + probe_validate_packet_cb validate_packet; + probe_classify_packet_cb process_packet; + probe_close_cb close; + int output_type; + fielddef_t * fields; + int numfields; + const char * helptext; + +} probe_module_t; + +probe_module_t *get_probe_module_by_name(const char *name, int ipv46_flag); + +void fs_add_ip_fields(fieldset_t *fs, struct ip *ip); + +void fs_add_ipv6_fields(fieldset_t *fs, struct ip6_hdr *ipv6_hdr); + +void fs_add_system_fields(fieldset_t *fs, int is_repeat, int in_cooldown); + +void print_probe_modules(int ipv46_flag); + +extern int ip_fields_len; +extern int ip6_fields_len; +extern int sys_fields_len; +extern fielddef_t ip_fields[]; +extern fielddef_t ip6_fields[]; +extern fielddef_t sys_fields[]; + +#endif // XMAP_PROBE_MODULES_H diff --git a/src/recv-internal.h b/src/recv-internal.h new file mode 100644 index 0000000..476706e --- /dev/null +++ b/src/recv-internal.h @@ -0,0 +1,27 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_RECV_INTERNAL_H +#define XMAP_RECV_INTERNAL_H + +#include +#include + +void handle_packet(uint32_t buflen, const uint8_t *bytes, + const struct timespec ts); + +void recv_init(); + +void recv_packets(); + +void recv_cleanup(); + +#endif // XMAP_RECV_INTERNAL_H diff --git a/src/recv-pcap.c b/src/recv-pcap.c new file mode 100644 index 0000000..49649cb --- /dev/null +++ b/src/recv-pcap.c @@ -0,0 +1,156 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv-internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined __linux__ && __linux__ +#include +#endif + +#include "probe_modules/probe_modules.h" +#include "state.h" + +#include "../lib/logger.h" + +#define PCAP_PROMISC 1 +#define PCAP_TIMEOUT 1000 + +static pcap_t *pc = NULL; + +void packet_cb(u_char __attribute__((__unused__)) * user, + const struct pcap_pkthdr *p, const u_char *bytes) { + struct timespec ts; + if (!p) { + return; + } + + if (xrecv.filter_success >= xconf.max_results) { + // Libpcap can process multiple packets per pcap_dispatch; + // we need to throw out results once we've + // gotten our --max-results worth. + return; + } + + // length of entire packet captured by libpcap + uint32_t buflen = (uint32_t) p->caplen; + ts.tv_sec = p->ts.tv_sec; + ts.tv_nsec = p->ts.tv_usec * 1000; + handle_packet(buflen, bytes, ts); +} + +#define BPFLEN 1024 + +void recv_init() { + char bpftmp[BPFLEN]; + char errbuf[PCAP_ERRBUF_SIZE]; + + pc = pcap_open_live(xconf.iface, xconf.probe_module->pcap_snaplen, + PCAP_PROMISC, PCAP_TIMEOUT, errbuf); + if (pc == NULL) { + log_fatal("recv", "could not open device %s: %s", xconf.iface, errbuf); + } + switch (pcap_datalink(pc)) { + case DLT_EN10MB: + log_info("recv", "Data link layer Ethernet"); + xconf.data_link_size = sizeof(struct ether_header); + break; + case DLT_RAW: + log_info("recv", "Data link RAW"); + xconf.data_link_size = 0; + break; +#if defined __linux__ && __linux__ + case DLT_LINUX_SLL: + log_info("recv", "Data link cooked socket"); + xconf.data_link_size = SLL_HDR_LEN; + break; +#endif + default: + log_error("recv", "unknown data link layer"); + } + + struct bpf_program bpf; + + if (!xconf.send_ip_pkts) { + snprintf(bpftmp, sizeof(bpftmp) - 1, + "not ether src %02x:%02x:%02x:%02x:%02x:%02x", xconf.hw_mac[0], + xconf.hw_mac[1], xconf.hw_mac[2], xconf.hw_mac[3], + xconf.hw_mac[4], xconf.hw_mac[5]); + assert(strlen(xconf.probe_module->pcap_filter) + 10 < + (BPFLEN - strlen(bpftmp))); + } else { + bpftmp[0] = 0; + } + if (xconf.probe_module->pcap_filter) { + if (!xconf.send_ip_pkts) { + strcat(bpftmp, " and ("); + } else { + strcat(bpftmp, "("); + } + strcat(bpftmp, xconf.probe_module->pcap_filter); + strcat(bpftmp, ")"); + } + if (strcmp(bpftmp, "") != 0) { + if (pcap_compile(pc, &bpf, bpftmp, 1, 0) < 0) { + log_fatal("recv", "couldn't compile bpf filter"); + } + if (pcap_setfilter(pc, &bpf) < 0) { + log_fatal("recv", "couldn't install bpf filter"); + } + } + // set pcap_dispatch to not hang if it never receives any packets + // this could occur if you ever scan a small number of hosts as + // documented in issue #74. + if (pcap_setnonblock(pc, 1, errbuf) == -1) { + log_fatal("recv", "pcap_setnonblock error:%s", errbuf); + } +} + +void recv_packets() { + int ret = pcap_dispatch(pc, -1, packet_cb, NULL); + if (ret == -1) { + log_fatal("recv", "pcap_dispatch error"); + } else if (ret == 0) { + usleep(1000); + } +} + +void recv_cleanup() { + pcap_close(pc); + pc = NULL; +} + +int recv_update_stats(void) { + if (!pc) { + return EXIT_FAILURE; + } + + struct pcap_stat pcst; + if (pcap_stats(pc, &pcst)) { + log_error("recv", "unable to retrieve pcap statistics: %s", + pcap_geterr(pc)); + return EXIT_FAILURE; + } else { + xrecv.pcap_recv = pcst.ps_recv; + xrecv.pcap_drop = pcst.ps_drop; + xrecv.pcap_ifdrop = pcst.ps_ifdrop; + } + + return EXIT_SUCCESS; +} diff --git a/src/recv-pfring.c b/src/recv-pfring.c new file mode 100644 index 0000000..d85cea9 --- /dev/null +++ b/src/recv-pfring.c @@ -0,0 +1,86 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv-internal.h" +#include "recv.h" + +#include +#include +#include + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "state.h" + +static pfring_zc_pkt_buff *pf_buffer; +static pfring_zc_queue *pf_recv; + +void recv_init() { + // Get the socket and packet handle + pf_recv = xconf.pf.recv; + pf_buffer = pfring_zc_get_packet_handle(xconf.pf.cluster); + + if (pf_buffer == NULL) { + log_fatal("recv", "could not get pfring packet handle: %s", strerror(errno)); + } + + xconf.data_link_size = sizeof(struct ether_header); // TODO case? +} + +void recv_cleanup() { + if (!pf_recv) { + return; + } + + pfring_zc_sync_queue(pf_recv, rx_only); +} + +void recv_packets() { + int ret; + + // Poll for packets + ret = pfring_zc_recv_pkt(pf_recv, &pf_buffer, 0); + if (ret == 0) { + usleep(1000); + return; + } + + // Handle other errors, by not doing anything and logging + if (ret != 1) { + log_error("recv", "pfring error: %d", ret); + return; + } + + // Successfully got a packet, now handle it + struct timespec ts; + ts.tv_sec = pf_buffer->ts.tv_sec; + ts.tv_nsec = pf_buffer->ts.tv_nsec; + + uint8_t *pkt_buf = pfring_zc_pkt_buff_data(pf_buffer, pf_recv); + handle_packet(pf_buffer->len, pkt_buf, ts); +} + +int recv_update_stats(void) { + if (!pf_recv) { + return EXIT_FAILURE; + } + + pfring_zc_stat pfst; + if (pfring_zc_stats(pf_recv, &pfst)) { + log_error("recv", "unable to retrieve pfring statistics"); + return EXIT_FAILURE; + } else { + xrecv.pcap_recv = pfst.recv; + xrecv.pcap_drop = pfst.drop; + } + + return EXIT_SUCCESS; +} diff --git a/src/recv.c b/src/recv.c new file mode 100644 index 0000000..125d9e0 --- /dev/null +++ b/src/recv.c @@ -0,0 +1,215 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "recv.h" + +#include +#include +#include + +#include "fieldset.h" +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" +#include "recv-internal.h" +#include "state.h" +#include "validate.h" + +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/util.h" + +static u_char fake_eth_hdr[65535]; + +void handle_packet(uint32_t buflen, const u_char *bytes, + const struct timespec ts) { + struct ip * ip_header; + struct ip6_hdr *ip6_header; + + if (xconf.ipv46_flag == IPV6_FLAG) { + if ((sizeof(struct ip6_hdr) + xconf.data_link_size) > buflen) { + // buffer not large enough to contain ethernet + // and ipv6 headers. further action would overrun buf + return; + } + ip6_header = (struct ip6_hdr *) &bytes[xconf.data_link_size]; + ip_header = (struct ip *) ip6_header; + } else { + if ((sizeof(struct ip) + xconf.data_link_size) > buflen) { + // buffer not large enough to contain ethernet + // and ip headers. further action would overrun buf + return; + } + ip_header = (struct ip *) &bytes[xconf.data_link_size]; + } + + int is_repeat = 1; + if (!xconf.probe_module->validate_packet( + ip_header, + buflen - (xconf.send_ip_pkts ? 0 : sizeof(struct ether_header)), + &is_repeat)) { + xrecv.validation_failed++; + return; + } else { + xrecv.validation_passed++; + } + + // track whether this is the first packet in an IP fragment. + if (xconf.ipv46_flag == IPV4_FLAG) { + if (ip_header->ip_off & (u_short)(IP_MF)) { + xrecv.ip_fragments++; + } + } + + fieldset_t *fs = fs_new_fieldset(); + + if (xconf.ipv46_flag == IPV6_FLAG) { + fs_add_ipv6_fields(fs, ip6_header); + } else { + fs_add_ip_fields(fs, ip_header); + } + + // HACK: + // probe modules expect the full ethernet frame + // in process_packet. For VPN, we only get back an IP frame. + // Here, we fake an ethernet frame (which is initialized to + // have ETH_P_IP proto and 00s for dest/src). + if (xconf.send_ip_pkts) { + if (buflen > sizeof(fake_eth_hdr)) { + buflen = sizeof(fake_eth_hdr); + } + memcpy(&fake_eth_hdr[sizeof(struct ether_header)], + bytes + xconf.data_link_size, buflen); + bytes = fake_eth_hdr; + } + xconf.probe_module->process_packet(bytes, buflen, fs, ts); + fs_add_system_fields(fs, is_repeat, xsend.complete); + + int success_index = xconf.fsconf.success_index; + assert(success_index < fs->len); + int is_success = fs_get_uint64_by_index(fs, success_index); + + if (is_success) { + xrecv.success_total++; + if (!is_repeat) { + xrecv.success_unique++; + } + if (xsend.complete) { + xrecv.cooldown_total++; + if (!is_repeat) { + xrecv.cooldown_unique++; + } + } + } else { + xrecv.failure_total++; + } + // probe module includes app_success field + if (xconf.fsconf.app_success_index >= 0) { + int is_app_success = + fs_get_uint64_by_index(fs, xconf.fsconf.app_success_index); + if (is_app_success) { + xrecv.app_success_total++; + if (!is_repeat) { + xrecv.app_success_unique++; + } + } + } + + fieldset_t *o = NULL; + // we need to translate the data provided by the probe module + // into a fieldset that can be used by the output module + if (!is_success && xconf.filter_unsuccessful) { + goto cleanup; + } + if (is_repeat && xconf.filter_duplicates) { + goto cleanup; + } + if (!evaluate_expression(xconf.filter.expression, fs)) { + goto cleanup; + } + xrecv.filter_success++; + o = translate_fieldset(fs, &xconf.fsconf.translation); + if (xconf.output_module && xconf.output_module->process_ip) { + xconf.output_module->process_ip(o); + } + +cleanup: + fs_free(fs); + free(o); + if (xconf.output_module && xconf.output_module->update && + !(xrecv.success_unique % xconf.output_module->update_interval)) { + xconf.output_module->update(&xconf, &xsend, &xrecv); + } +} + +int recv_run(pthread_mutex_t *recv_ready_mutex) { + log_debug("recv", "recv thread started"); + log_debug("recv", "capturing responses on %s", xconf.iface); + + if (!xconf.dryrun) { + recv_init(); + } + + if (xconf.send_ip_pkts) { + struct ether_header *eth = (struct ether_header *) fake_eth_hdr; + memset(fake_eth_hdr, 0, sizeof(fake_eth_hdr)); + if (xconf.ipv46_flag == IPV4_FLAG) + eth->ether_type = htons(ETHERTYPE_IP); + else + eth->ether_type = htons(ETHERTYPE_IPV6); + } + + if (xconf.filter_duplicates) { + log_debug("recv", "duplicate responses will be excluded from output"); + } else { + log_debug("recv", "duplicate responses will be included in output"); + } + if (xconf.filter_unsuccessful) { + log_debug("recv", + "unsuccessful responses will be excluded from output"); + } else { + log_debug("recv", "unsuccessful responses will be included in output"); + } + + pthread_mutex_lock(recv_ready_mutex); + xconf.recv_ready = 1; + pthread_mutex_unlock(recv_ready_mutex); + xrecv.start = now(); + if (xconf.max_results == 0) { + xconf.max_results = -1; + } + + do { + if (xconf.dryrun) { + sleep(1); + } else { + recv_packets(); + if (xconf.max_results && + xrecv.filter_success >= xconf.max_results) { + break; + } + } + } while (!(xsend.complete && (now() - xsend.finish > xconf.cooldown_secs))); + + xrecv.finish = now(); + + // get final pcap statistics before closing + recv_update_stats(); + if (!xconf.dryrun) { + pthread_mutex_lock(recv_ready_mutex); + recv_cleanup(); + pthread_mutex_unlock(recv_ready_mutex); + } + + xrecv.complete = 1; + log_debug("recv", "thread finished"); + + return 0; +} diff --git a/src/recv.h b/src/recv.h new file mode 100644 index 0000000..713f01a --- /dev/null +++ b/src/recv.h @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_RECV_H +#define XMAP_RECV_H + +#include + +int recv_update_stats(void); + +int recv_run(pthread_mutex_t *recv_ready_mutex); + +#endif // XMAP_RECV_H diff --git a/src/send-bsd.h b/src/send-bsd.h new file mode 100644 index 0000000..92acc2a --- /dev/null +++ b/src/send-bsd.h @@ -0,0 +1,35 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SEND_BSD_H +#define XMAP_SEND_BSD_H + +#include +#include + +#include "../lib/includes.h" + +#ifdef XMAP_SEND_LINUX_H +#error "Don't include both send-bsd.h and send-linux.h" +#endif + +// thread +int send_run_init(UNUSED sock_t sock) { + // Don't need to do anything on BSD-like variants + return EXIT_SUCCESS; +} + +// thread +int send_packet(sock_t sock, void *buf, int len, UNUSED uint32_t idx) { + return write(sock.sock, buf, len); +} + +#endif // XMAP_SEND_BSD_H diff --git a/src/send-linux.h b/src/send-linux.h new file mode 100644 index 0000000..a642d10 --- /dev/null +++ b/src/send-linux.h @@ -0,0 +1,71 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SEND_LINUX_H +#define XMAP_SEND_LINUX_H + +#include +#include + +#include "../lib/includes.h" +#include "../lib/util.h" + +#ifdef XMAP_SEND_BSD_H +#error "Don't include both send-bsd.h and send-linux.h" +#endif + +// Dummy sockaddr for sendto in link level +static struct sockaddr_ll sockaddr; + +int send_run_init(sock_t s) { + // Get the actual socket + int sock = s.sock; + // get source interface index + struct ifreq if_idx; + memset(&if_idx, 0, sizeof(struct ifreq)); + if (strlen(xconf.iface) >= IFNAMSIZ) { + log_error("send", "device interface name (%s) too long\n", xconf.iface); + return EXIT_FAILURE; + } + strncpy(if_idx.ifr_name, xconf.iface, IFNAMSIZ - 1); + if (ioctl(sock, SIOCGIFINDEX, &if_idx) < 0) { + perror("SIOCGIFINDEX"); + return EXIT_FAILURE; + } + int ifindex = if_idx.ifr_ifindex; + + // destination address for the socket + memset((void *) &sockaddr, 0, sizeof(struct sockaddr_ll)); + sockaddr.sll_ifindex = ifindex; + sockaddr.sll_halen = ETH_ALEN; + if (xconf.send_ip_pkts) { + switch (xconf.ipv46_flag) { + case IPV6_FLAG: + sockaddr.sll_protocol = htons(ETHERTYPE_IPV6); + break; + case IPV4_FLAG: + sockaddr.sll_protocol = htons(ETHERTYPE_IP); + break; + default: + log_fatal("socket", "iplayer for linux not supported on IPv%d", + xconf.ipv46_flag); + } + } + memcpy(sockaddr.sll_addr, xconf.gw_mac, ETH_ALEN); + return EXIT_SUCCESS; +} + +int send_packet(sock_t sock, void *buf, int len, UNUSED uint32_t idx) { + return sendto(sock.sock, buf, len, 0, (struct sockaddr *) &sockaddr, + sizeof(struct sockaddr_ll)); +} + +#endif //XMAP_SEND_LINUX_H diff --git a/src/send-pfring.h b/src/send-pfring.h new file mode 100644 index 0000000..90494bd --- /dev/null +++ b/src/send-pfring.h @@ -0,0 +1,46 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SEND_PFRING_H +#define XMAP_SEND_PFRING_H + +#include "../lib/includes.h" +#include + +#if defined(XMAP_SEND_BSD_H) || defined(XMAP_SEND_LINUX_H) +#error "Don't include send-bsd.h or send-linux.h with send-pfring.h" +#endif + +int send_run_init(sock_t socket) { + (void) socket; + // All init for pfring happens in get_socket + + return 0; +} + +int send_packet(sock_t sock, void *buf, int len, uint32_t idx) { + sock.pf.buffers[idx]->len = len; + memcpy(pfring_zc_pkt_buff_data(sock.pf.buffers[idx], sock.pf.queue), + buf, len); + + int ret; + do { + ret = pfring_zc_send_pkt(sock.pf.queue, &(sock.pf.buffers[idx]), 0); + } while (ret < 0); + + pfring_zc_sync_queue(sock.pf.queue, tx_only); + + return ret; +} + +void send_finish(sock_t sock) { pfring_zc_sync_queue(sock.pf.queue, tx_only); } + +#endif //XMAP_SEND_PFRING_H diff --git a/src/send.c b/src/send.c new file mode 100644 index 0000000..8353a64 --- /dev/null +++ b/src/send.c @@ -0,0 +1,505 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "send.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "iid_modules/iid_modules.h" +#include "ip_target_file.h" +#include "probe_modules/packet.h" +#include "probe_modules/probe_modules.h" +#include "shard.h" +#include "state.h" +#include "validate.h" + +#include "../lib/blocklist.h" +#include "../lib/gmp-ext.h" +#include "../lib/lockfd.h" +#include "../lib/logger.h" +#include "../lib/util.h" + +// OS specific functions called by send_run +static inline int send_packet(sock_t sock, void *buf, int len, uint32_t idx); + +static inline int send_run_init(sock_t sock); + +// Include the right implementations +#if defined(PFRING) +#include "send-pfring.h" +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__DragonFly__) +#include "send-bsd.h" +#else // LINUX +#include "send-linux.h" +#endif // __APPLE__ || __FreeBSD__ || __NetBSD__ || __DragonFly__ + +// The iterator over the cyclic group + +// Lock for send run +static pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER; + +// Source IP address offset for outgoing packets +static uint32_t srcip_offset; + +// Source ports for outgoing packets +static uint16_t num_src_ports; + +void sig_handler_increase_speed(UNUSED int signal) { + int old_rate = xconf.rate; + xconf.rate += (xconf.rate * 0.05); + log_info("send", "send rate increased from %i to %i pps.", old_rate, + xconf.rate); +} + +void sig_handler_decrease_speed(UNUSED int signal) { + int old_rate = xconf.rate; + xconf.rate -= (xconf.rate * 0.05); + log_info("send", "send rate decreased from %i to %i pps.", old_rate, + xconf.rate); +} + +// global sender initialize (not thread specific) +iterator_t *send_init(void) { + log_debug("send", "send_init for global send start."); + + // only allow bandwidth or rate + if (xconf.bandwidth > 0 && xconf.rate > 0) { + log_fatal("send", + "must specify rate or bandwidth, or neither, not both."); + } + // convert specified bandwidth to packet rate + if (xconf.bandwidth > 0) { + size_t pkt_len = xconf.probe_module->packet_length; + pkt_len *= 8; + // 7 byte MAC preamble, 1 byte Start frame, 4 byte CRC, 12 byte + // inter-frame gap + pkt_len += 8 * 24; + // adjust calculated length if less than the minimum size of an + // ethernet frame + if (pkt_len < 84 * 8) { + pkt_len = 84 * 8; + } + // rate is a uint32_t so, don't overflow + if (xconf.bandwidth / pkt_len > 0xFFFFFFFFu) { + xconf.rate = 0; + } else { + xconf.rate = xconf.bandwidth / pkt_len; + if (xconf.rate == 0) { + log_warn("send", + "bandwidth %lu bit/s is slower than 1 pkt/s, setting " + "rate to 1 pkt/s", + xconf.bandwidth); + xconf.rate = 1; + } + } + log_debug("send", + "using bandwidth %lu bits/s for %zu byte probe, rate set to " + "%d pkt/s", + xconf.bandwidth, pkt_len / 8, xconf.rate); + } + // convert default placeholder to default value + if (xconf.rate == -1) { + // default 1 pps + xconf.rate = 1; + } + // log rate, if explicitly specified + if (xconf.rate < 0) { + log_fatal("send", "rate impossibly slow"); + } + if (xconf.rate > 0 && xconf.bandwidth <= 0) { + size_t pkt_len = xconf.probe_module->packet_length; + pkt_len *= 8; + // 7 byte MAC preamble, 1 byte Start frame, 4 byte CRC, 12 byte + // inter-frame gap + pkt_len += 8 * 24; + // adjust calculated length if less than the minimum size of an + // ethernet frame + if (pkt_len < 84 * 8) { + pkt_len = 84 * 8; + } + xconf.bandwidth = xconf.rate * pkt_len; + } + + char rate_str[20]; + char bd_str[20]; + number_string(xconf.rate, rate_str, 20); + bits_string(xconf.bandwidth, bd_str, 20); + log_debug("send", "rate set to %sp/s", rate_str); + log_debug("send", "bandwidth set to %sb/s", bd_str); + + // automatically set sender-thread-num + /* + if ((int) ceil(xconf.rate / 1000000.0) > xconf.senders) { + xconf.senders = (int) ceil(xconf.rate / 1000000.0); + log_warn("send", + "sender thread number is not enough to send on this rate=%d " + "\n\t\t\t\t (for GMP lib computing performance), automatically + " "set sender-thread-num to %d", xconf.rate, xconf.senders); if + (xconf.senders > xconf.pin_cores_len) log_fatal("send", "sender thread num > + core num"); + }*/ + + // generate a new primitive root and starting position + iterator_t *it; + uint32_t num_subshards = + (uint32_t) xconf.senders * (uint32_t) xconf.total_shards; + mpz_t block_allowed_ip_port; + mpz_init(block_allowed_ip_port); + blocklist_count_allowed_ip_port(block_allowed_ip_port); + if (mpz_lt_ui(block_allowed_ip_port, num_subshards)) { + log_fatal("send", "senders * shards > allowed probing address"); + } + if (xsend.max_targets && (num_subshards > xsend.max_targets)) { + log_fatal("send", "senders * shards > max targets"); + } + + it = iterator_init(xconf.senders, xconf.shard_num, xconf.total_shards); + + // determine the source address offset from which we'll send packets + char *temp_addr = + inet_in2constr(xconf.source_ip_addresses[0], xconf.ipv46_flag); + log_debug("send", "src_ipv%d_1st: %s", xconf.ipv46_flag, temp_addr); + temp_addr = + inet_in2constr(xconf.source_ip_addresses[xconf.number_source_ips - 1], + xconf.ipv46_flag); + log_debug("send", "src_ipv%d_1st: %s", xconf.ipv46_flag, temp_addr); + if (xconf.number_source_ips == 1) { + srcip_offset = 0; + } else { + uint32_t offset = (uint32_t)(aesrand_getword(xconf.aes) & 0xFFFFFFFF); + srcip_offset = offset % (xconf.number_source_ips); + } + + // process the source port range that XMap is allowed to use + num_src_ports = xconf.source_port_last - xconf.source_port_first + 1; + log_debug("send", "xmap will send from %i address%s on %u source ports", + xconf.number_source_ips, + ((xconf.number_source_ips == 1) ? "" : "es"), num_src_ports); + + // global initialization for send module + assert(xconf.probe_module); + if (xconf.probe_module->global_init) { + if (xconf.probe_module->global_init(&xconf)) { + log_fatal("send", + "global initialization for probe module (%s) failed", + xconf.probe_module->name); + } + log_debug("send", "probe module global initialize"); + } + + // global initialization for iid module + assert(xconf.iid_module); + if (xconf.iid_module->global_init) { + if (xconf.iid_module->global_init(&xconf)) { + log_fatal("send", "global initialization for iid module failed."); + } + log_debug("send", "iid module global initialize"); + } + + // just generate IP address + if (xconf.dryrun) { + log_info("send", "dryrun mode -- won't actually send packets"); + } + + // initialize random validation key + validate_init(); + + // setup signal handlers for changing scan speed + signal(SIGUSR1, sig_handler_increase_speed); + signal(SIGUSR2, sig_handler_decrease_speed); + + // start + xsend.start = now(); + + mpz_clear(block_allowed_ip_port); + free(temp_addr); + + log_debug("send", "send_init for global send completed."); + + return it; +} + +static inline uint8_t *get_src_ip(const uint8_t *dst, int local_offset) { + if (xconf.number_source_ips == 1) { + return xconf.source_ip_addresses[0]; + } + + uint8_t offset = srcip_offset + local_offset; + int i; + for (i = 0; i < xconf.ipv46_bytes; i++) + offset += dst[i]; + offset %= xconf.number_source_ips; + + return xconf.source_ip_addresses[offset]; +} + +// one sender thread +int send_run(sock_t st, shard_t *sd) { + log_debug("send", "send thread started"); + pthread_mutex_lock(&send_mutex); + + // Allocate a buffer to hold the outgoing packet + char buff[MAX_PACKET_SIZE]; + memset(buff, 0, MAX_PACKET_SIZE); + + // OS specific per-thread init + if (send_run_init(st)) { + return -1; + } + + // probe thread initialize + void *probe_data; + if (xconf.probe_module->thread_init) + xconf.probe_module->thread_init(buff, xconf.hw_mac, xconf.gw_mac, + &probe_data); + + // iid thread initialize + if (xconf.iid_module->thread_init) xconf.iid_module->thread_init(); + + pthread_mutex_unlock(&send_mutex); + + // adaptive timing to hit target rate + uint64_t count = 0; + uint64_t last_count = count; + uint32_t delay = 0; + int interval = 0; + volatile int vi; + struct timespec ts, rem; + double send_rate = + (double) xconf.rate / ((double) xconf.senders * xconf.batch); + const double slow_rate = 50; // packets per seconds per thread + // at which it uses the slow methods + long nsec_per_sec = 1000 * 1000 * 1000; + long long sleep_time = nsec_per_sec; + double last_time = now(); + if (xconf.rate > 0) { + delay = 10000; + if (send_rate < slow_rate) { + // set the inital time difference + sleep_time = (double) nsec_per_sec / send_rate; + last_time = now() - (1.0 / send_rate); + } else { + // estimate initial rate, sleep for 1/rate time needs how many delay + // times + for (vi = delay; vi--;) + ; + delay *= + 1 / (now() - last_time) / + ((double) xconf.rate / ((double) xconf.senders * xconf.batch)); + interval = (int) (((double) xconf.rate / + ((double) xconf.senders * xconf.batch)) / + 20); + last_time = now(); + } + } + + // Get the initial IP to scan. + ipaddr_n_t dst_ip[xconf.ipv46_bytes]; + port_h_t dst_port; + memset(dst_ip, 0, xconf.ipv46_bytes); + if (xconf.list_of_ips_filename) { + if (ip_target_file_get_ip(dst_ip, sd) == EXIT_FAILURE) + memset(dst_ip, 0, xconf.ipv46_bytes); + dst_port = ip_target_file_get_port(sd); + } else + dst_port = shard_get_current_ip_prefix_port(dst_ip, sd); + + ipaddr_n_t current_ip_suffix[xconf.ipv46_bytes]; + uint32_t idx = 0; // pfring buffer index + mpz_t temp; // temp using + mpz_init(temp); + + log_debug("send", "1st scanned IPv%d prefix: %s", xconf.ipv46_flag, + inet_in2constr(dst_ip, xconf.ipv46_flag)); + + int b = 0; + // while for sending + while (1) { + // Check if we've finished this shard or thread before sending each + // packet, regardless of batch size. + if (sd->state.max_hosts && + sd->state.hosts_scanned >= sd->state.max_hosts) { + log_debug("send", + "send thread %hhu finished (max targets of %u reached)", + sd->thread_id, sd->state.max_hosts); + goto cleanup; + } + mpz_from_uint8s(temp, dst_ip, xconf.ipv46_bytes); + if (mpz_eq_ui(temp, PMAP_SHARD_DONE)) { + log_debug("send", "send thread %hhu finished, shard depleted", + sd->thread_id); + goto cleanup; + } + + // generate iid_num iid for per prefix + for (int u = 0; u < xconf.iid_num; u++) { + if (!xconf.list_of_ips_filename) { // not get ip from file + xconf.iid_module->get_current_iid(current_ip_suffix, u, NULL); + for (int k = 0; k < xconf.ipv46_bytes; k++) + dst_ip[k] = dst_ip[k] | current_ip_suffix[k]; + } + + // send packet_streams number of packet to per dst address + for (int i = 0; i < xconf.packet_streams; i++) { + // Check if the program has otherwise completed and break out of + // the send loop. + if (xrecv.complete) { + goto cleanup; + } + if (xconf.max_runtime && + xconf.max_runtime <= now() - xsend.start) { + goto cleanup; + } + if (sd->state.max_packets && + sd->state.packets_sent >= sd->state.max_packets) { + log_debug( + "send", + "send thread %hhu finished (max packets of %u reached)", + sd->thread_id, sd->state.max_packets); + goto cleanup; + } + + count++; + ipaddr_n_t *src_ip = get_src_ip(dst_ip, i); + uint8_t ttl = xconf.probe_ttl; + size_t length = xconf.probe_module->packet_length; + xconf.probe_module->make_packet(buff, &length, src_ip, dst_ip, + dst_port, ttl, i, probe_data); + if (length > MAX_PACKET_SIZE) { + log_fatal("send", + "send thread %hhu set length (%zu) larger than " + "MAX (%zu)", + sd->thread_id, length, MAX_PACKET_SIZE); + } + + // sleeping, maybe send batch before sleeping + if (b >= xconf.batch) { + // Adaptive timing delay + send_rate = (double) xconf.rate / + ((double) xconf.senders * xconf.batch); + if (count && delay > 0) { + if (send_rate < slow_rate) { + double t = now(); + double last_rate = (1.0 / (t - last_time)); + sleep_time *= ((last_rate / send_rate) + 1) / 2; + ts.tv_sec = sleep_time / nsec_per_sec; + ts.tv_nsec = sleep_time % nsec_per_sec; + log_debug("sleep", + "sleep for %d sec, %ld nanoseconds", + ts.tv_sec, ts.tv_nsec); + while (nanosleep(&ts, &rem) == -1) { + } + last_time = t; + } else { + for (vi = delay; vi--;) + ; + if (!interval || (count % interval == 0)) { + double t = now(); + assert(count > last_count); + assert(t > last_time); + double multiplier = + (double) (count - last_count) / + (t - last_time) / + (xconf.rate / xconf.senders); + uint32_t old_delay = delay; + delay *= multiplier; + if (delay == old_delay) { + if (multiplier > 1.0) { + delay *= 2; + } else if (multiplier < 1.0) { + delay *= 0.5; + } + } + last_count = count; + last_time = t; + } + } + } + b = 1; + } else { + b++; + } + + if (xconf.dryrun) { // just generating packet + lock_file(stdout); + xconf.probe_module->print_packet(stdout, buff); + unlock_file(stdout); + } else { + void *contents = + buff + + xconf.send_ip_pkts * + sizeof(struct ether_header); // only send IP packet + length -= + (xconf.send_ip_pkts * sizeof(struct ether_header)); + int any_sends_successful = 0; + + // sending + // send one packet for attempts times until one being + // successful + for (int j = 0; j < xconf.num_retries; j++) { + // log_info("send", "IP=%s", inet_in2constr(dst_ip, + // xconf.ipv46_flag)); + int rc = + send_packet(st, contents, length, + idx); // idx for pfring buffer index + sd->state.packets_tried++; + if (rc < 0) { // failed + char addr_str[64]; + inet_in2str(dst_ip, addr_str, 64, 6); + log_debug("send", "send_packet failed for %s. %s", + addr_str, strerror(errno)); + } else { + any_sends_successful = 1; + break; + } + } + if (!any_sends_successful) { + sd->state.packets_failed++; + } + idx++; + idx &= 0xFFu; // shifting pfring buffer index + } + + sd->state.packets_sent++; + } + } + + // Track the number of hosts we actually scanned. + sd->state.hosts_scanned++; + + // Get the next IP to scan + if (xconf.list_of_ips_filename) { + if (ip_target_file_get_ip(dst_ip, sd) == EXIT_FAILURE) + memset(dst_ip, 0, xconf.ipv46_bytes); + dst_port = ip_target_file_get_port(sd); + } else + dst_port = shard_get_next_ip_prefix_port(dst_ip, sd); + } + +cleanup: + sd->completeCb(sd->thread_id, sd->args); + if (xconf.dryrun) { + lock_file(stdout); + fflush(stdout); + unlock_file(stdout); + } + mpz_clear(temp); + log_debug("send", "thread %hu cleanly finished", sd->thread_id); + + return EXIT_SUCCESS; +} diff --git a/src/send.h b/src/send.h new file mode 100644 index 0000000..4d22aef --- /dev/null +++ b/src/send.h @@ -0,0 +1,24 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SEND_H +#define XMAP_SEND_H + +#include "iterator.h" +#include "socket.h" + +// global sender initialize (not thread specific) +iterator_t *send_init(void); + +// one sender thread +int send_run(sock_t, shard_t *); + +#endif // XMAP_SEND_H diff --git a/src/shard.c b/src/shard.c new file mode 100644 index 0000000..4213c2f --- /dev/null +++ b/src/shard.c @@ -0,0 +1,281 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "shard.h" + +#include +#include + +#include "state.h" + +#include "../lib/blocklist.h" +#include "../lib/gmp-ext.h" +#include "../lib/logger.h" + +static inline void shard_get_next_elem(shard_t *shard) { + // no overflow + mpz_mul(shard->current, shard->current, shard->params.factor); + mpz_mod(shard->current, shard->current, shard->params.modulus); +} + +static void _shard_get_current_ipvx_port(mpz_t ipvx, shard_t *shard) { + mpz_t index; + mpz_init(index); + mpz_sub_ui(index, shard->current, 1); + blocklist_lookup_index_for_ipvx_port(ipvx, index); + mpz_clear(index); +} + +static void _shard_get_next_ipvx_port(mpz_t ipvx, shard_t *shard) { + if (mpz_eq_ui(shard->current, PMAP_SHARD_DONE)) { + mpz_set_ui(ipvx, PMAP_SHARD_DONE); + return; + } + + mpz_t index; + mpz_init(index); + while (1) { + shard_get_next_elem(shard); + if (mpz_eq(shard->current, shard->params.last)) { + mpz_set_ui(shard->current, PMAP_SHARD_DONE); + shard->iterations++; + mpz_set_ui(ipvx, PMAP_SHARD_DONE); + goto cleanup; + } + mpz_sub_ui(index, shard->current, 1); + if (mpz_lt(index, xsend.max_index)) { + shard->state.hosts_allowlisted++; + shard->iterations++; + blocklist_lookup_index_for_ipvx_port(ipvx, index); + goto cleanup; + } + shard->state.hosts_blocklisted++; + } + +cleanup: + mpz_clear(index); +} + +static void _shard_roll_to_valid(mpz_t ipvx, shard_t *shard) { + if (mpz_le(shard->current, xsend.max_index)) { + mpz_set(ipvx, shard->current); + return; + } + _shard_get_next_ipvx_port(ipvx, shard); +} + +void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards, + uint8_t thread_idx, uint8_t num_threads, + uint64_t max_total_targets, uint64_t max_total_packets, + uint64_t list_of_ips_count, const cycle_t *cycle, + shard_complete_cb cb, void *args) { + // Start out by figuring out how many shards we have. A single shard of + // XMap (set with --shards=N, --shard=n) may have several subshards, if + // XMap is being ran multithreaded (set with --sender-threads=T). + // + // Total number of subshards is S = N*T. Subshard ID's range from [0, N*T). + log_debug("shard", "shard_init: shard-%d", thread_idx); + + assert(num_shards > 0); + assert(num_threads > 0); + assert(shard_idx < num_shards); + assert(thread_idx < num_threads); + uint32_t num_subshards = (uint32_t) num_shards * (uint32_t) num_threads; + mpz_t num_elts; + mpz_init_set(num_elts, cycle->order); + assert(mpz_gt_ui(num_elts, num_subshards)); // in case to many shards + assert(!max_total_targets || max_total_targets >= num_subshards); + assert(!max_total_packets || max_total_packets >= num_subshards); + assert(!list_of_ips_count || list_of_ips_count >= num_subshards); + + // This instance of XMap will run T subshards, with one subshard per + // thread. This composes a single shard, as specified by the command + // line flag --shard=n. E.g. to run shard with index n, we must run + // subshards with indicies the range [n*T, (n+1)*T]. + // + // We can calculate our subshard index i = n*T + t. + uint32_t sub_idx = shard_idx * num_threads + thread_idx; + + // Given i, we want to calculate the start of subshard i. Subshards + // define ranges over exponents of g. They range from [0, Q-1), where Q + // is the number of elements in (the order of) the group generated by g. + // + // Let e_b = floor(Q / S) * i + mpz_t exponent_begin; + mpz_init(exponent_begin); + mpz_fdiv_q_ui(exponent_begin, num_elts, num_subshards); + mpz_mul_ui(exponent_begin, exponent_begin, sub_idx); + + // The stopping exponent is the first element of the next shard. + // + // e_e = floor(Q / S) * ((i + 1) % S); + mpz_t exponent_end; + mpz_init(exponent_end); + mpz_fdiv_q_ui(exponent_end, num_elts, num_subshards); + mpz_mul_ui(exponent_end, exponent_end, (sub_idx + 1) % num_subshards); + + // We actually offset the begin and end of each cycle. Given an offset + // k, shift each exponent by k modulo Q. + mpz_add(exponent_begin, exponent_begin, cycle->offset); + mpz_mod(exponent_begin, exponent_begin, num_elts); + mpz_add(exponent_end, exponent_end, cycle->offset); + mpz_mod(exponent_end, exponent_end, num_elts); + + // Calculate the first and last points of the shard as powers of g + // modulo p. g^i mod p (i: index in a range) + mpz_t start, stop; + mpz_init(start); + mpz_init(stop); + mpz_powm(start, cycle->generator, exponent_begin, cycle->group->prime); + mpz_powm(stop, cycle->generator, exponent_end, cycle->group->prime); + + // Pull the result out + mpz_init_set(shard->params.first, start); + mpz_init_set(shard->params.last, stop); + mpz_init_set(shard->params.factor, cycle->generator); + mpz_init_set(shard->params.modulus, cycle->group->prime); + + // Set the shard at the beginning. + mpz_init_set(shard->current, shard->params.first); + + // Set the (thread) id + shard->thread_id = thread_idx; + + // Set max_targets if applicable + if (max_total_targets > 0) { + uint64_t max_targets_this_shard = max_total_targets / num_subshards; + if (sub_idx < (max_total_targets % num_subshards)) + ++max_targets_this_shard; + shard->state.max_hosts = max_targets_this_shard; + } + + // Set max_packets if applicable + if (max_total_packets > 0) { + uint64_t max_packets_this_shard = max_total_packets / num_subshards; + if (sub_idx < (max_total_packets % num_subshards)) + ++max_packets_this_shard; + shard->state.max_packets = max_packets_this_shard; + } + + // initializing ip_target_file + shard->ip_target_file_params.fp = NULL; + if (list_of_ips_count) { + uint64_t ip_target_this_shard = list_of_ips_count / num_subshards; + shard->ip_target_file_params.first = sub_idx * ip_target_this_shard; + if (sub_idx < (list_of_ips_count % num_subshards)) { + ++ip_target_this_shard; + shard->ip_target_file_params.first += sub_idx; + } else { + shard->ip_target_file_params.first += + (list_of_ips_count % num_subshards); + } + shard->ip_target_file_params.current = + shard->ip_target_file_params.first; + shard->ip_target_file_params.total = ip_target_this_shard; + shard->ip_target_file_params.last = shard->ip_target_file_params.first + + shard->ip_target_file_params.total; + shard->ip_target_file_params.fp = + fopen(xconf.list_of_ips_filename, "r"); + if (shard->ip_target_file_params.fp == NULL) { + log_fatal("shard", "open ip target file for shard failed"); + } + shard->ip_target_file_params.port_current = 0; + shard->ip_target_file_params.port_total = xconf.target_port_num; + } + + // Set the callbacks + shard->completeCb = cb; + shard->args = args; + + // If the beginning of a shard isn't pointing to a valid index in the + // blocklist, find the first element that is. + // num_elts just take a position + _shard_roll_to_valid(num_elts, shard); + + // initializing other + mpz_init_set(shard->state.first_scanned, shard->current); + + // Clear everything + mpz_clear(num_elts); + mpz_clear(exponent_begin); + mpz_clear(exponent_end); + mpz_clear(start); + mpz_clear(stop); + + log_debug("shard", "shard_init: shard-%d completed", thread_idx); +} + +port_h_t shard_get_current_ip_prefix_port(void *prefix, shard_t *shard) { + mpz_t ipvx_port, ipvx, port_m; + mpz_init(ipvx_port); + mpz_init(ipvx); + mpz_init(port_m); + + _shard_get_current_ipvx_port(ipvx_port, shard); + mpz_mod_2exp(port_m, ipvx_port, xconf.target_port_bits); + + port_h_t port = (port_h_t) mpz_get_ui(port_m); + while (mpz_ne_ui(ipvx_port, PMAP_SHARD_DONE) && xconf.target_port_num && + port >= xconf.target_port_num) { + _shard_get_next_ipvx_port(ipvx_port, shard); + mpz_mod_2exp(port_m, ipvx_port, xconf.target_port_bits); + port = (port_h_t) mpz_get_ui(port_m); + } + + mpz_fdiv_q_2exp(ipvx, ipvx_port, xconf.target_port_bits); + memset(prefix, 0, xconf.ipv46_bytes); + mpz_to_uint8s_bits(ipvx, prefix, xconf.max_probe_len); + + mpz_clear(ipvx_port); + mpz_clear(ipvx); + mpz_clear(port_m); + + return xconf.target_port_list[port]; +} + +port_h_t shard_get_next_ip_prefix_port(void *prefix, shard_t *shard) { + mpz_t ipvx_port, ipvx, port_m; + mpz_init(ipvx_port); + mpz_init(ipvx); + mpz_init(port_m); + + _shard_get_next_ipvx_port(ipvx_port, shard); + mpz_mod_2exp(port_m, ipvx_port, xconf.target_port_bits); + + port_h_t port = (port_h_t) mpz_get_ui(port_m); + while (mpz_ne_ui(ipvx_port, PMAP_SHARD_DONE) && xconf.target_port_num && + port >= xconf.target_port_num) { + _shard_get_next_ipvx_port(ipvx_port, shard); + mpz_mod_2exp(port_m, ipvx_port, xconf.target_port_bits); + port = (port_h_t) mpz_get_ui(port_m); + } + + mpz_fdiv_q_2exp(ipvx, ipvx_port, xconf.target_port_bits); + memset(prefix, 0, xconf.ipv46_bytes); + mpz_to_uint8s_bits(ipvx, prefix, xconf.max_probe_len); + + mpz_clear(ipvx_port); + mpz_clear(ipvx); + mpz_clear(port_m); + + return xconf.target_port_list[port]; +} + +void shard_free(shard_t *shard) { + mpz_clear(shard->state.first_scanned); + mpz_clear(shard->params.first); + mpz_clear(shard->params.last); + mpz_clear(shard->params.factor); + mpz_clear(shard->params.modulus); + if (shard->ip_target_file_params.fp) + fclose(shard->ip_target_file_params.fp); + mpz_clear(shard->current); +} diff --git a/src/shard.h b/src/shard.h new file mode 100644 index 0000000..26ce0c1 --- /dev/null +++ b/src/shard.h @@ -0,0 +1,73 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SHARD_H +#define XMAP_SHARD_H + +#include +#include +#include + +#include "../lib/types.h" +#include "cyclic.h" + +#define PMAP_SHARD_DONE 0 + +typedef void (*shard_complete_cb)(uint8_t id, void *args); + +typedef struct shard { + struct shard_state { + uint64_t packets_sent; + uint64_t hosts_scanned; + uint64_t max_hosts; + uint64_t max_packets; + uint64_t hosts_blocklisted; + uint64_t hosts_allowlisted; + uint64_t packets_failed; + uint64_t packets_tried; + mpz_t first_scanned; + } state; + struct shard_params { + mpz_t first; + mpz_t last; + mpz_t factor; + mpz_t modulus; + } params; + struct shard_ip_target_file_params { + uint64_t first; + uint64_t last; + uint64_t current; + uint64_t total; + FILE * fp; + long pos; + uint16_t port_current; + uint16_t port_total; + } ip_target_file_params; + mpz_t current; + uint64_t iterations; + uint8_t thread_id; + shard_complete_cb completeCb; + void * args; +} shard_t; + +void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards, + uint8_t thread_idx, uint8_t num_threads, + uint64_t max_total_targets, uint64_t max_total_packets, + uint64_t list_of_ips_count, const cycle_t *cycle, + shard_complete_cb cb, void *args); + +port_h_t shard_get_current_ip_prefix_port(void *prefix, shard_t *shard); + +port_h_t shard_get_next_ip_prefix_port(void *prefix, shard_t *shard); + +void shard_free(shard_t *shard); + +#endif // XMAP_SHARD_H diff --git a/src/socket-bsd.c b/src/socket-bsd.c new file mode 100644 index 0000000..f54bb01 --- /dev/null +++ b/src/socket-bsd.c @@ -0,0 +1,67 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include +#include +#include +#include +#include +#include + +#include "state.h" + +#include "../lib/includes.h" +#include "../lib/logger.h" + +sock_t get_socket(UNUSED uint32_t id) { + char file[32]; + int bpf; + // Assume failure + sock_t ret; + ret.sock = -1; + + if (xconf.send_ip_pkts && !xconf.dryrun) { + log_fatal("socket", "iplayer not supported on bsd"); + } + + // Try to find a valid bpf + for (int i = 0; i < 128; i++) { + snprintf(file, sizeof(file), "/dev/bpf%d", i); + bpf = open(file, O_WRONLY); + if (bpf != -1 || errno != EBUSY) break; + } + + // Make sure it worked + if (bpf < 0) { + return ret; + } + + // Set up an ifreq to bind to + struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); + strlcpy(ifr.ifr_name, xconf.iface, sizeof(ifr.ifr_name)); + + // Bind the bpf to the interface + if (ioctl(bpf, BIOCSETIF, (char *) &ifr) < 0) { + return ret; + } + + // Enable writing the address in + int write_addr_enable = 1; + if (ioctl(bpf, BIOCSHDRCMPLT, &write_addr_enable) < 0) { + return ret; + } + ret.sock = bpf; + + return ret; +} diff --git a/src/socket-linux.c b/src/socket-linux.c new file mode 100644 index 0000000..e7d7c68 --- /dev/null +++ b/src/socket-linux.c @@ -0,0 +1,40 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include +#include + +#include "state.h" + +#include "../lib/includes.h" +#include "../lib/logger.h" + +sock_t get_socket(UNUSED uint32_t id) { + int sock; + if (xconf.send_ip_pkts) { + sock = socket(AF_PACKET, SOCK_DGRAM, + htons(ETH_P_ALL)); // link level for all ipv46- + } else { + sock = socket(AF_PACKET, SOCK_RAW, + htons(ETH_P_ALL)); // link level for all eth-ipv46- + } + if (sock <= 0) { + log_fatal("send", + "couldn't create socket. " + "Are you root? Error: %s\n", + strerror(errno)); + } + sock_t s; + s.sock = sock; + return s; +} diff --git a/src/socket-pfring.c b/src/socket-pfring.c new file mode 100644 index 0000000..4a2107d --- /dev/null +++ b/src/socket-pfring.c @@ -0,0 +1,25 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include + +#include "state.h" + +#include "../lib/includes.h" + +sock_t get_socket(uint32_t id) { + sock_t sock; + sock.pf.queue = xconf.pf.queues[id]; + sock.pf.buffers = xconf.pf.buffers + 256 * id; + return sock; +} diff --git a/src/socket.c b/src/socket.c new file mode 100644 index 0000000..bdd57da --- /dev/null +++ b/src/socket.c @@ -0,0 +1,33 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "socket.h" + +#include +#include + +#include "../lib/logger.h" + +sock_t get_dryrun_socket(void) { + // we need a socket in order to gather details about the system + // such as source MAC address and IP address. However, because + // we don't want to require root access in order to run dryrun, + // we just create a TCP socket. + int sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock <= 0) { + log_fatal("send", "couldn't create socket. Error: %s\n", + strerror(errno)); + } + sock_t s; + s.sock = sock; + + return s; +} diff --git a/src/socket.h b/src/socket.h new file mode 100644 index 0000000..e26cd44 --- /dev/null +++ b/src/socket.h @@ -0,0 +1,42 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SOCKET_H +#define XMAP_SOCKET_H + +#include + +#include "../lib/includes.h" + +#ifdef PFRING +#include + +typedef union { + int sock; + struct { + pfring_zc_queue * queue; + pfring_zc_pkt_buff **buffers; + int idx; + } pf; +} sock_t; + +#else + +typedef struct { + int sock; +} sock_t; +#endif // PFRING + +sock_t get_dryrun_socket(void); + +sock_t get_socket(uint32_t id); + +#endif // XMAP_SOCKET_H diff --git a/src/state.c b/src/state.c new file mode 100644 index 0000000..fce2737 --- /dev/null +++ b/src/state.c @@ -0,0 +1,120 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "state.h" + +#include "../lib/logger.h" + +// global configuration and defaults +struct state_conf xconf = { + .ipv46_flag = 6, + .max_probe_len = 32, + .target_port_list = {0}, + .target_port_flag = {0}, + .target_port_num = 0, + .target_port_bits = 0, + .target_port_full = 1, + .max_probe_port_len = 32, + .source_port_first = 32768, // (these are the default + .source_port_last = 61000, // ephemeral range on Linux) + .iface = NULL, + .gw_mac = {0x00}, + .hw_mac = {0x00}, + .gw_ip = {0x00}, + .gw_mac_set = 0, + .hw_mac_set = 0, + .source_ip_addresses = NULL, + .number_source_ips = 0, + .send_ip_pkts = 0, + .rate = -1, + .bandwidth = 0, + .batch = 1, + .packet_streams = 1, + .cooldown_secs = 5, + .max_targets = 0, + .max_packets = 0, + .max_runtime = 0, + .max_results = 0, + .est_elements = 5e8, + .senders = 1, + .seed_provided = 0, + .seed = 0, + .probe_module = NULL, + .probe_args = NULL, + .probe_ttl = MAXTTL, + .output_module = NULL, + .output_args = NULL, + .iid_module = NULL, + .iid_args = NULL, + .iid_num = 1, + .output_filename = NULL, + .blocklist_filename = NULL, + .allowlist_filename = NULL, + .list_of_ips_filename = NULL, + .list_of_ip_port_count = 0, + .metadata_filename = NULL, + .metadata_file = NULL, + .notes = NULL, + .custom_metadata_str = NULL, + .raw_output_fields = NULL, + .output_fields = NULL, + .output_filter_str = NULL, + .output_fields_len = 0, + .log_level = XLOG_INFO, + .syslog = 0, + .log_file = NULL, + .log_directory = NULL, + .status_updates_file = NULL, + .dryrun = 0, + .quiet = 0, + .filter_duplicates = 0, + .filter_unsuccessful = 0, + .max_sendto_failures = -1, + .recv_ready = 0, + .min_hitrate = (float) 0.0, + .data_link_size = 0, + .config_filename = NULL, +}; + +// global sender stats and defaults +struct state_send xsend = { + .start = 0.0, + .finish = 0.0, + .packets_sent = 0, + .hosts_scanned = 0, + .blocklisted = 0, + .allowlisted = 0, + .warmup = 1, + .complete = 0, + .max_targets = 0, + .max_packets = 0, + .sendto_failures = 0, + .packets_tried = 0, +}; + +// global receiver stats and defaults +struct state_recv xrecv = { + .success_unique = 0, + .success_total = 0, + .app_success_unique = 0, + .app_success_total = 0, + .validation_passed = 0, + .validation_failed = 0, + .cooldown_unique = 0, + .cooldown_total = 0, + .failure_total = 0, + .filter_success = 0, + .ip_fragments = 0, + .complete = 0, + .pcap_recv = 0, + .pcap_drop = 0, + .pcap_ifdrop = 0, +}; diff --git a/src/state.h b/src/state.h new file mode 100644 index 0000000..9bc41f4 --- /dev/null +++ b/src/state.h @@ -0,0 +1,254 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_STATE_H +#define XMAP_STATE_H + +#include +#include +#include + +#ifdef PFRING +#include +#endif + +#include "aesrand.h" +#include "fieldset.h" +#include "filter.h" + +#include "../lib/bloom.h" +#include "../lib/includes.h" +#include "../lib/types.h" + +#define MAX_PACKET_SIZE 4096 +#define MAC_ADDR_LEN_BYTES 6 +#define MAX_SOURCE_IP_NUM 1024 +#define IP_ADDR_LEN_BYTES 16 +#define MAX_PORT_NUM 65536 + +struct probe_module; +struct output_module; +struct iid_module; + +struct fieldset_conf { + fielddefset_t defs; + fielddefset_t outdefs; + translation_t translation; + int success_index; + int app_success_index; + int classification_index; +}; + +// global configuration +struct state_conf { + // IPv46 flag + int ipv46_flag; + int max_probe_len; + int ipv46_bytes; + int ipv46_bits; + + // target cidrs + char ** destination_cidrs; + int destination_cidrs_len; + port_h_t target_port_list[MAX_PORT_NUM]; + port_h_t target_port_flag[MAX_PORT_NUM]; + int target_port_num; + int target_port_bits; + int target_port_full; + int max_probe_port_len; + + port_h_t source_port_first; + port_h_t source_port_last; + // name of network interface that will be utilized for sending/receiving + char * iface; + macaddr_t gw_mac[MAC_ADDR_LEN_BYTES]; + macaddr_t hw_mac[MAC_ADDR_LEN_BYTES]; + ipaddr_n_t gw_ip[IP_ADDR_LEN_BYTES]; + int gw_mac_set; + int hw_mac_set; + ipaddr_n_t source_ip_addresses[MAX_SOURCE_IP_NUM][IP_ADDR_LEN_BYTES]; + uint32_t number_source_ips; + // send ip packet instead of ethernet packet + int send_ip_pkts; + + // rate in packets per second that the sender will maintain + int rate; + // rate in bits per second + uint64_t bandwidth; + uint8_t batch; + int packet_streams; + int num_retries; + // how many seconds after the termination of the sender will the receiver + // continue to process responses + int cooldown_secs; + + // maximum number of targets that the scanner will probe before terminating + uint64_t max_targets; + // maximum number of packets that the scanner will send before + // terminating + uint64_t max_packets; + // maximum number of seconds that scanner will run before terminating + uint32_t max_runtime; + // maximum number of results before terminating + uint64_t max_results; + // estimated elements of scanning for unique + uint64_t est_elements; + + // number of sending threads + uint8_t senders; + uint32_t pin_cores_len; + uint32_t *pin_cores; + // should use CLI provided randomization seed instead of generating a random + // seed. + int seed_provided; + uint64_t seed; + aesrand_t *aes; + // generator of the cyclic multiplicative group that is utilized for address + // generation + mpz_t generator; + // sharding options + uint16_t shard_num; + uint16_t total_shards; + + // probe module + struct probe_module *probe_module; + char * probe_args; + uint8_t probe_ttl; + // output module + struct output_module *output_module; + char * output_module_name; + char * output_args; + // IID value + struct iid_module *iid_module; + char * iid_args; + int iid_num; + char * iid_module_name; + + // file + char * output_filename; + char * blocklist_filename; + char * allowlist_filename; + char * list_of_ips_filename; + uint64_t list_of_ip_count; + uint64_t list_of_ip_port_count; + char * metadata_filename; + FILE * metadata_file; + char * notes; + char * custom_metadata_str; + + // output field + char * raw_output_fields; + char ** output_fields; + struct output_filter filter; + char * output_filter_str; + struct fieldset_conf fsconf; + int output_fields_len; + + // log & other + int log_level; + char * log_file; + char * log_directory; + char * status_updates_file; + int dryrun; + int quiet; + int ignore_blacklist_error; + int ignore_filelist_error; + int syslog; + int filter_duplicates; + int filter_unsuccessful; + int recv_ready; + uint64_t max_sendto_failures; + float min_hitrate; + int data_link_size; + char * config_filename; + + mpz_t total_allowed_ip_port; + mpz_t total_disallowed_ip_port; + mpz_t total_allowed_ip; + mpz_t total_disallowed_ip; + mpz_t total_allowed_ip_port_actual; + +#ifdef PFRING + struct { + pfring_zc_cluster * cluster; + pfring_zc_queue * send; + pfring_zc_queue * recv; + pfring_zc_queue ** queues; + pfring_zc_pkt_buff ** buffers; + pfring_zc_buffer_pool *prefetches; + } pf; +#endif +}; +extern struct state_conf xconf; + +// global sender stats +struct state_send { + double start; + double finish; + uint64_t packets_sent; + uint64_t hosts_scanned; + uint64_t blocklisted; + uint64_t allowlisted; + int warmup; + int complete; + uint64_t max_targets; + uint64_t max_packets; + uint64_t sendto_failures; + uint64_t packets_tried; + mpz_t first_scanned; + mpz_t max_index; // max index for send +}; +extern struct state_send xsend; + +// global receiver stats +struct state_recv { + // valid responses classified as "success" + uint64_t success_total; + // unique IPs that sent valid responses classified as "success" + uint64_t success_unique; + // valid responses classified as "success" + uint64_t app_success_total; + // unique IPs that sent valid responses classified as "success" + uint64_t app_success_unique; + // valid responses classified as "success" received during cooldown + uint64_t cooldown_total; + // unique IPs that first sent valid "success"es during cooldown + uint64_t cooldown_unique; + // valid responses NOT classified as "success" + uint64_t failure_total; + // valid responses that passed the filter + uint64_t filter_success; + // how many packets did we receive that were marked as being the first + // fragment in a stream + uint64_t ip_fragments; + // metrics about _only_ validate_packet + uint64_t validation_passed; + uint64_t validation_failed; + + int complete; // has the scanner finished sending? + double start; // timestamp of when recv started + double finish; // timestamp of when recv terminated + + // number of packets captured by pcap filter + uint64_t pcap_recv; + // number of packets dropped because there was no room in the operating + // system's buffer when they arrived, because packets weren't being read + // fast enough + uint64_t pcap_drop; + // number of packets dropped by the network interface or its driver. + uint64_t pcap_ifdrop; + + // used for repeating check + BloomFilter bf; +}; +extern struct state_recv xrecv; + +#endif // XMAP_STATE_H diff --git a/src/summary.c b/src/summary.c new file mode 100644 index 0000000..6cecace --- /dev/null +++ b/src/summary.c @@ -0,0 +1,431 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "summary.h" + +#include +#include +#include +#include +#include +#include + +#include "iid_modules/iid_modules.h" +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" +#include "state.h" + +#include "../lib/blocklist.h" +#include "../lib/gmp-ext.h" +#include "../lib/includes.h" +#include "../lib/logger.h" +#include "../lib/util.h" + +#define STRTIME_LEN 1024 + +#include + +void json_metadata(FILE *file) { + char send_start_time[STRTIME_LEN + 1]; + assert(dstrftime(send_start_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + xsend.start)); + char send_end_time[STRTIME_LEN + 1]; + assert(dstrftime(send_end_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + xsend.finish)); + char recv_start_time[STRTIME_LEN + 1]; + assert(dstrftime(recv_start_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + xrecv.start)); + char recv_end_time[STRTIME_LEN + 1]; + assert(dstrftime(recv_end_time, STRTIME_LEN, "%Y-%m-%dT%H:%M:%S%z", + xrecv.finish)); + double hitrate = + ((double) 100 * xrecv.success_unique) / ((double) xsend.hosts_scanned); + + json_object *obj = json_object_new_object(); + + // scanner host name + char hostname[1024]; + if (gethostname(hostname, 1023) < 0) { + log_error("json_metadata", "unable to retrieve local hostname"); + } else { + hostname[1023] = '\0'; + json_object_object_add(obj, "local_hostname", + json_object_new_string(hostname)); + struct hostent *h = gethostbyname(hostname); + if (h) { + json_object_object_add(obj, "full_hostname", + json_object_new_string(h->h_name)); + } else { + log_error("json_metadata", "unable to retrieve complete hostname"); + } + } + + json_object_object_add(obj, "ipv46_flag", + json_object_new_int(xconf.ipv46_flag)); + json_object_object_add(obj, "max_probe_len", + json_object_new_int(xconf.max_probe_len)); + json_object_object_add(obj, "max_probe_port_len", + json_object_new_int(xconf.max_probe_port_len)); + json_object_object_add(obj, "target_port_num", + json_object_new_int(xconf.target_port_num)); + json_object_object_add(obj, "target_port_bits", + json_object_new_int(xconf.target_port_bits)); + if (xconf.target_port_num) { + json_object *target_ports = json_object_new_array(); + for (int i = 0; i < xconf.target_port_num; i++) { + json_object_array_add( + target_ports, json_object_new_int(xconf.target_port_list[i])); + } + json_object_object_add(obj, "target_ports", target_ports); + } + json_object_object_add(obj, "source_port_first", + json_object_new_int(xconf.source_port_first)); + json_object_object_add(obj, "source_port_last", + json_object_new_int(xconf.source_port_last)); + json_object_object_add(obj, "max_targets", + json_object_new_int64(xconf.max_targets)); + json_object_object_add(obj, "max_packets", + json_object_new_int64(xconf.max_packets)); + json_object_object_add(obj, "max_runtime", + json_object_new_int(xconf.max_runtime)); + json_object_object_add(obj, "max_results", + json_object_new_int64(xconf.max_results)); + json_object_object_add(obj, "est_elements", + json_object_new_int64(xconf.est_elements)); + json_object_object_add(obj, "output_results", + json_object_new_int64(xrecv.filter_success)); + if (xconf.iface) { + json_object_object_add(obj, "iface", + json_object_new_string(xconf.iface)); + } + char rate_str[20]; + char bd_str[20]; + number_string(xconf.rate, rate_str, 20); + bits_string(xconf.bandwidth, bd_str, 20); + strcat(rate_str, "pps"); + strcat(bd_str, "bps"); + json_object_object_add(obj, "rate", json_object_new_string(rate_str)); + json_object_object_add(obj, "bandwidth", json_object_new_string(bd_str)); + json_object_object_add(obj, "batch", json_object_new_int(xconf.batch)); + json_object_object_add(obj, "probes", + json_object_new_int(xconf.packet_streams)); + json_object_object_add(obj, "retires", + json_object_new_int(xconf.num_retries)); + json_object_object_add(obj, "iid_num", json_object_new_int(xconf.iid_num)); + json_object_object_add(obj, "cooldown_secs", + json_object_new_int(xconf.cooldown_secs)); + json_object_object_add(obj, "senders", json_object_new_int(xconf.senders)); + if (xconf.pin_cores_len) { + json_object *pin_cores = json_object_new_array(); + for (uint32_t i = 0; i < xconf.pin_cores_len; i++) { + json_object_array_add(pin_cores, + json_object_new_int(xconf.pin_cores[i])); + } + json_object_object_add(obj, "pin_cores", pin_cores); + } + json_object_object_add(obj, "seed", json_object_new_int64(xconf.seed)); + json_object_object_add(obj, "seed_provided", + json_object_new_int(xconf.seed_provided)); + json_object_object_add( + obj, "generator", + json_object_new_string(mpz_to_str10(xconf.generator))); + json_object_object_add(obj, "hitrate", json_object_new_double(hitrate)); + json_object_object_add(obj, "shard_num", + json_object_new_int(xconf.shard_num)); + json_object_object_add(obj, "total_shards", + json_object_new_int(xconf.total_shards)); + + json_object_object_add(obj, "min_hitrate", + json_object_new_double(xconf.min_hitrate)); + json_object_object_add(obj, "max_sendto_failures", + json_object_new_int64(xconf.max_sendto_failures)); + + json_object_object_add(obj, "syslog", json_object_new_int(xconf.syslog)); + json_object_object_add(obj, "filter_duplicates", + json_object_new_int(xconf.filter_duplicates)); + json_object_object_add(obj, "filter_unsuccessful", + json_object_new_int(xconf.filter_unsuccessful)); + + json_object_object_add(obj, "pcap_recv", + json_object_new_int64(xrecv.pcap_recv)); + json_object_object_add(obj, "pcap_drop", + json_object_new_int64(xrecv.pcap_drop)); + json_object_object_add(obj, "pcap_ifdrop", + json_object_new_int64(xrecv.pcap_ifdrop)); + + json_object_object_add(obj, "ip_fragments", + json_object_new_int64(xrecv.ip_fragments)); + json_object_object_add( + obj, "blocklist_total_allowed_ip", + json_object_new_string(mpz_to_str10(xconf.total_allowed_ip))); + json_object_object_add( + obj, "blocklist_total_not_allowed_ip", + json_object_new_string(mpz_to_str10(xconf.total_disallowed_ip))); + json_object_object_add(obj, "blocklist_total_allowed_ip_port_actual", + json_object_new_string(mpz_to_str10( + xconf.total_allowed_ip_port_actual))); + json_object_object_add( + obj, "blocklist_total_allowed_ip_port", + json_object_new_string(mpz_to_str10(xconf.total_allowed_ip_port))); + json_object_object_add( + obj, "blocklist_total_not_allowed_ip_port", + json_object_new_string(mpz_to_str10(xconf.total_disallowed_ip_port))); + json_object_object_add(obj, "validation_passed", + json_object_new_int64(xrecv.validation_passed)); + json_object_object_add(obj, "validation_failed", + json_object_new_int64(xrecv.validation_failed)); + + // json_object_object_add(obj, "blocklisted", + // json_object_new_int64(xsend.blocklisted)); + // json_object_object_add(obj, "allowlisted", + // json_object_new_int64(xsend.allowlisted)); + uint8_t ip[16]; + memset(ip, 0, 16); + mpz_t ip_1; + mpz_init(ip_1); + mpz_sub_ui(xsend.first_scanned, xsend.first_scanned, 1); + blocklist_lookup_index_for_ipvx_port(ip_1, xsend.first_scanned); + mpz_to_uint8s_bits(ip_1, ip, xconf.max_probe_len); + json_object_object_add( + obj, "first_scanned_prefix", + json_object_new_string(inet_in2constr(ip, xconf.ipv46_flag))); + mpz_clear(ip_1); + + json_object_object_add( + obj, "first_scanned", + json_object_new_string(mpz_to_str10(xsend.first_scanned))); + json_object_object_add(obj, "send_to_failures", + json_object_new_int64(xsend.sendto_failures)); + json_object_object_add(obj, "packets_sent", + json_object_new_int64(xsend.packets_sent)); + json_object_object_add(obj, "packets_tried", + json_object_new_int64(xsend.packets_tried)); + json_object_object_add(obj, "hosts_scanned", + json_object_new_int64(xsend.hosts_scanned)); + json_object_object_add(obj, "success_total", + json_object_new_int64(xrecv.success_total)); + json_object_object_add(obj, "success_unique", + json_object_new_int64(xrecv.success_unique)); + if (xconf.fsconf.app_success_index >= 0) { + json_object_object_add(obj, "app_success_total", + json_object_new_int64(xrecv.app_success_total)); + json_object_object_add(obj, "app_success_unique", + json_object_new_int64(xrecv.app_success_unique)); + } + json_object_object_add(obj, "success_cooldown_total", + json_object_new_int64(xrecv.cooldown_total)); + json_object_object_add(obj, "success_cooldown_unique", + json_object_new_int64(xrecv.cooldown_unique)); + json_object_object_add(obj, "failure_total", + json_object_new_int64(xrecv.failure_total)); + + json_object_object_add( + obj, "probe_module", + json_object_new_string(((probe_module_t *) xconf.probe_module)->name)); + json_object_object_add( + obj, "output_module", + json_object_new_string( + ((output_module_t *) xconf.output_module)->name)); + json_object_object_add( + obj, "iid_module", + json_object_new_string(((iid_module_t *) xconf.iid_module)->name)); + + json_object_object_add(obj, "send_start_time", + json_object_new_string(send_start_time)); + json_object_object_add(obj, "send_end_time", + json_object_new_string(send_end_time)); + json_object_object_add(obj, "recv_start_time", + json_object_new_string(recv_start_time)); + json_object_object_add(obj, "recv_end_time", + json_object_new_string(recv_end_time)); + + if (xconf.output_filter_str) { + json_object_object_add(obj, "output_filter", + json_object_new_string(xconf.output_filter_str)); + } + if (xconf.log_file) { + json_object_object_add(obj, "log_file", + json_object_new_string(xconf.log_file)); + } + if (xconf.log_directory) { + json_object_object_add(obj, "log_directory", + json_object_new_string(xconf.log_directory)); + } + + if (xconf.destination_cidrs_len) { + json_object *cli_dest_cidrs = json_object_new_array(); + for (int i = 0; i < xconf.destination_cidrs_len; i++) { + json_object_array_add( + cli_dest_cidrs, + json_object_new_string(xconf.destination_cidrs[i])); + } + json_object_object_add(obj, "cli_cidr_destinations", cli_dest_cidrs); + } + if (xconf.probe_args) { + json_object_object_add(obj, "probe_args", + json_object_new_string(xconf.probe_args)); + } + if (xconf.probe_ttl) { + json_object_object_add(obj, "probe_ttl", + json_object_new_int(xconf.probe_ttl)); + } + if (xconf.output_args) { + json_object_object_add(obj, "output_args", + json_object_new_string(xconf.output_args)); + } + if (xconf.iid_args) { + json_object_object_add(obj, "iid_args", + json_object_new_string(xconf.iid_args)); + } + { + char mac_buf[(MAC_ADDR_LEN * 2) + (MAC_ADDR_LEN - 1) + 1]; + memset(mac_buf, 0, sizeof(mac_buf)); + char *p = mac_buf; + for (int i = 0; i < MAC_ADDR_LEN; i++) { + if (i == MAC_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", xconf.gw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", xconf.gw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "gateway_mac", + json_object_new_string(mac_buf)); + } + if (xconf.gw_ip) { + json_object_object_add(obj, "gateway_ip", + json_object_new_string(inet_in2constr( + xconf.gw_ip, xconf.ipv46_flag))); + } + { + char mac_buf[(ETHER_ADDR_LEN * 2) + (ETHER_ADDR_LEN - 1) + 1]; + char *p = mac_buf; + for (int i = 0; i < ETHER_ADDR_LEN; i++) { + if (i == ETHER_ADDR_LEN - 1) { + snprintf(p, 3, "%.2x", xconf.hw_mac[i]); + p += 2; + } else { + snprintf(p, 4, "%.2x:", xconf.hw_mac[i]); + p += 3; + } + } + json_object_object_add(obj, "source_mac", + json_object_new_string(mac_buf)); + } + json_object *source_ips = json_object_new_array(); + for (uint i = 0; i < xconf.number_source_ips; i++) { + json_object_array_add( + source_ips, json_object_new_string(inet_in2constr( + xconf.source_ip_addresses[i], xconf.ipv46_flag))); + } + json_object_object_add(obj, "source_ips", source_ips); + json_object_object_add(obj, "iplayer", + json_object_new_int(xconf.send_ip_pkts)); + if (xconf.output_filename) { + json_object_object_add(obj, "output_filename", + json_object_new_string(xconf.output_filename)); + } + if (xconf.blocklist_filename) { + json_object_object_add( + obj, "blocklist_filename", + json_object_new_string(xconf.blocklist_filename)); + } + if (xconf.allowlist_filename) { + json_object_object_add( + obj, "allowlist_filename", + json_object_new_string(xconf.allowlist_filename)); + } + if (xconf.list_of_ips_filename) { + json_object_object_add( + obj, "list_of_ips_filename", + json_object_new_string(xconf.list_of_ips_filename)); + json_object_object_add(obj, "list_of_ip_count", + json_object_new_int(xconf.list_of_ip_count)); + json_object_object_add( + obj, "list_of_ip_port_count", + json_object_new_int(xconf.list_of_ip_port_count)); + } + json_object_object_add(obj, "ignore_blacklist_error", + json_object_new_int(xconf.ignore_blacklist_error)); + json_object_object_add(obj, "ignore_filelist_error", + json_object_new_int(xconf.ignore_filelist_error)); + if (xconf.config_filename) { + json_object_object_add(obj, "config_filename", + json_object_new_string(xconf.config_filename)); + } + if (xconf.status_updates_file) { + json_object_object_add( + obj, "status_updates_file", + json_object_new_string(xconf.status_updates_file)); + } + if (xconf.metadata_filename) { + json_object_object_add(obj, "metadata_filename", + json_object_new_string(xconf.metadata_filename)); + } + json_object_object_add(obj, "dryrun", json_object_new_int(xconf.dryrun)); + json_object_object_add(obj, "quiet", json_object_new_int(xconf.quiet)); + json_object_object_add(obj, "log_level", + json_object_new_int(xconf.log_level)); + + // parse out JSON metadata that was supplied on the command-line + if (xconf.custom_metadata_str) { + json_object *user = json_tokener_parse(xconf.custom_metadata_str); + if (!user) { + log_error("json-metadata", "unable to parse user metadata"); + } else { + json_object_object_add(obj, "user-metadata", user); + } + } + + if (xconf.notes) { + json_object_object_add(obj, "notes", + json_object_new_string(xconf.notes)); + } + + // add blocklisted and allowlisted CIDR blocks + bl_cidr_node_t *b = get_blocklisted_cidrs(); + if (b) { + json_object *blocklisted_cidrs = json_object_new_array(); + char cidr[64]; + uint8_t ip[16]; + do { + memset(cidr, 0, 64); + memset(ip, 0, 16); + mpz_to_uint8s_bits(b->ipvx_address, ip, xconf.max_probe_len); + sprintf(cidr, "%s/%i", inet_in2constr(ip, xconf.ipv46_flag), + b->prefix_len); + json_object_array_add(blocklisted_cidrs, + json_object_new_string(cidr)); + } while (b && (b = b->next)); + json_object_object_add(obj, "blocklisted_networks", blocklisted_cidrs); + } + + b = get_allowlisted_cidrs(); + if (b) { + json_object *allowlisted_cidrs = json_object_new_array(); + char cidr[64]; + uint8_t ip[16]; + do { + memset(cidr, 0, 64); + memset(ip, 0, 16); + mpz_to_uint8s_bits(b->ipvx_address, ip, xconf.max_probe_len); + sprintf(cidr, "%s/%i", inet_in2constr(ip, xconf.ipv46_flag), + b->prefix_len); + json_object_array_add(allowlisted_cidrs, + json_object_new_string(cidr)); + } while (b && (b = b->next)); + json_object_object_add(obj, "allowlisted_networks", allowlisted_cidrs); + } + + fprintf(file, "%s\n", json_object_to_json_string(obj)); + json_object_put(obj); +} diff --git a/src/summary.h b/src/summary.h new file mode 100644 index 0000000..dee2fc5 --- /dev/null +++ b/src/summary.h @@ -0,0 +1,19 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_SUMMARY_H +#define XMAP_SUMMARY_H + +#include + +void json_metadata(FILE *); + +#endif // XMAP_SUMMARY_H diff --git a/src/tests/xtest.c b/src/tests/xtest.c new file mode 100644 index 0000000..47d0c46 --- /dev/null +++ b/src/tests/xtest.c @@ -0,0 +1,36 @@ +/* + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "aesrand.h" +#include "cyclic.h" +#include "get_gateway.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "probe_modules/packet_icmp.h" +#include "probe_modules/packet_icmp6.h" + +#include "../lib/blocklist.h" +#include "../lib/bloom.h" +#include "../lib/constraint.h" +#include "../lib/gmp-ext.h" +#include "../lib/logger.h" +#include "../lib/xalloc.h" + +int main() { + return EXIT_SUCCESS; +} diff --git a/src/utility.c b/src/utility.c new file mode 100644 index 0000000..f674d83 --- /dev/null +++ b/src/utility.c @@ -0,0 +1,153 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "utility.h" + +#include + +#include "../lib/gmp-ext.h" +#include "../lib/logger.h" +#include "../lib/util.h" +#include "state.h" + +void add_to_array(const char *ip_str) { + if (xconf.number_source_ips >= MAX_SOURCE_IP_NUM) + log_fatal("parse", "over %d source IPv%d addresses provided", + MAX_SOURCE_IP_NUM, xconf.ipv46_flag); + log_debug("parse", "source IPv%d address: %s\n", xconf.ipv46_flag, ip_str); + inet_str2in(ip_str, xconf.source_ip_addresses[xconf.number_source_ips], + xconf.ipv46_flag); + xconf.number_source_ips++; +} + +void parse_source_ip_addresses(char given_string[]) { + char *dash = strchr(given_string, '-'); + char *comma = strchr(given_string, ','); + + if (dash && comma) { + *comma = '\0'; + parse_source_ip_addresses(given_string); + parse_source_ip_addresses(comma + 1); + } else if (comma) { + while (comma) { + *comma = '\0'; + add_to_array(given_string); + given_string = comma + 1; + comma = strchr(given_string, ','); + if (!comma) { + add_to_array(given_string); + } + } + } else if (dash) { // range + *dash = '\0'; + log_debug("parse", "IPv%d address start: %s", xconf.ipv46_flag, + given_string); + log_debug("parse", "IPv%d address end: %s", xconf.ipv46_flag, dash + 1); + + uint8_t ip_start[IP_ADDR_LEN_BYTES]; + uint8_t ip_end[IP_ADDR_LEN_BYTES]; + inet_str2in(given_string, ip_start, xconf.ipv46_flag); + inet_str2in(dash + 1, ip_end, xconf.ipv46_flag); + + mpz_t ip_start_m, ip_end_m; + mpz_init(ip_start_m); + mpz_init(ip_end_m); + mpz_from_uint8s(ip_start_m, ip_start, xconf.ipv46_bytes); + mpz_from_uint8s(ip_end_m, ip_end, xconf.ipv46_bytes); + + char ip_str[64]; + while (mpz_le(ip_start_m, ip_end_m)) { + if (xconf.number_source_ips >= MAX_SOURCE_IP_NUM) { + // log fatal here + log_fatal("parse", "over %d source IPv%d addresses provided", + MAX_SOURCE_IP_NUM, xconf.ipv46_flag); + } + mpz_to_uint8s(ip_start_m, ip_start, xconf.ipv46_bytes); + for (int i = 0; i < xconf.ipv46_bytes; i++) + xconf.source_ip_addresses[xconf.number_source_ips][i] = + ip_start[i]; + inet_in2str(ip_start, ip_str, 64, xconf.ipv46_flag); + log_debug("parse", "IPv%d address: %s", xconf.ipv46_flag, ip_str); + mpz_add_ui(ip_start_m, ip_start_m, 1); + } + + mpz_clear(ip_start_m); + mpz_clear(ip_end_m); + } else { + add_to_array(given_string); + } +} + +void add_to_target_port_array(int port) { + if (xconf.target_port_num >= MAX_PORT_NUM) + log_fatal("parse", "over %d target ports provided", MAX_PORT_NUM); + if (port < 0 || port >= MAX_PORT_NUM) + log_fatal("parse", "illegal target port: %d provided", port); + log_debug("parse", "target port: %d", port); + xconf.target_port_list[xconf.target_port_num] = port; + xconf.target_port_flag[port] = 1; + xconf.target_port_num++; +} + +void parse_target_ports(char given_string[]) { + char *dash = strchr(given_string, '-'); + char *comma = strchr(given_string, ','); + if (dash && comma) { + *comma = '\0'; + parse_target_ports(given_string); + parse_target_ports(comma + 1); + } else if (comma) { + while (comma) { + *comma = '\0'; + add_to_target_port_array(atoi(given_string)); + given_string = comma + 1; + comma = strchr(given_string, ','); + if (!comma) { + add_to_target_port_array(atoi(given_string)); + } + } + } else if (dash) { // range + *dash = '\0'; + int port_start = atoi(given_string); + int port_end = atoi(dash + 1); + + log_debug("parse", "target port start: %d", port_start); + log_debug("parse", "target port end: %d", port_end); + + while (port_start <= port_end) { + add_to_target_port_array(port_start); + port_start++; + } + } else { + add_to_target_port_array(atoi(given_string)); + } +} + +void init_target_port() { + log_debug("parse", "init target port"); + int full = 1; + for (int i = 1; i <= 16; i++) { + if (xconf.target_port_num <= full) { + log_debug("parse", "target port number: %d", xconf.target_port_num); + log_debug("parse", "target port bits: %d", xconf.target_port_bits); + log_debug("parse", "target port full range number: %d", + xconf.target_port_full); + log_debug("parse", "max_probe_port_len: %d", + xconf.max_probe_port_len); + return; + } + full *= 2; + xconf.target_port_bits = i; + xconf.target_port_full = full; + xconf.max_probe_port_len = xconf.max_probe_len + xconf.target_port_bits; + } + log_fatal("parse", "too many target ports (%d)", xconf.target_port_num); +} diff --git a/src/utility.h b/src/utility.h new file mode 100644 index 0000000..1547ee4 --- /dev/null +++ b/src/utility.h @@ -0,0 +1,21 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_UTILITY_H +#define XMAP_UTILITY_H + +void parse_source_ip_addresses(char given_string[]); + +void parse_target_ports(char given_string[]); + +void init_target_port(); + +#endif // XMAP_UTILITY_H diff --git a/src/validate.c b/src/validate.c new file mode 100644 index 0000000..1562a36 --- /dev/null +++ b/src/validate.c @@ -0,0 +1,53 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "validate.h" + +#include +#include + +#include "state.h" + +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/rijndael-alg-fst.h" + +#define AES_ROUNDS 10 +#define AES_BLOCK_WORDS 4 +#define AES_KEY_BYTES 16 + +static int inited = 0; +static uint32_t aes_sched[(AES_ROUNDS + 1) * 4]; + +void validate_init() { + uint8_t key[AES_KEY_BYTES]; + if (!random_bytes(key, AES_KEY_BYTES)) { + log_fatal("validate", "couldn't get random bytes"); + } + if (rijndaelKeySetupEnc(aes_sched, key, AES_KEY_BYTES * 8) != AES_ROUNDS) { + log_fatal("validate", "couldn't initialize AES key"); + } + inited = 1; +} + +void validate_gen(const uint8_t *src_ip, const uint8_t *dst_ip, + port_h_t dst_port, uint8_t output[VALIDATE_BYTES]) { + assert(inited); + + uint8_t aes_input[AES_KEY_BYTES]; + memset(aes_input, 0, AES_KEY_BYTES); + int i; + for (i = 0; i < xconf.ipv46_bytes; i++) + aes_input[i] = src_ip[i] ^ dst_ip[i]; + aes_input[0] ^= (uint8_t)(dst_port >> 8u); + aes_input[3] ^= (uint8_t)(dst_port & 0xffu); + rijndaelEncrypt(aes_sched, AES_ROUNDS, (uint8_t *) aes_input, output); +} diff --git a/src/validate.h b/src/validate.h new file mode 100644 index 0000000..49a20da --- /dev/null +++ b/src/validate.h @@ -0,0 +1,25 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#ifndef XMAP_VALIDATE_H +#define XMAP_VALIDATE_H + +#include "../lib/types.h" +#include + +#define VALIDATE_BYTES 16 + +void validate_init(); + +void validate_gen(const uint8_t *src_ip, const uint8_t *dst_ip, + port_h_t dst_port, uint8_t output[VALIDATE_BYTES]); + +#endif // XMAP_VALIDATE_H diff --git a/src/xmap.1 b/src/xmap.1 new file mode 100644 index 0000000..b47a6fa --- /dev/null +++ b/src/xmap.1 @@ -0,0 +1,353 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "XMAP" "1" "April 2021" "" "" +. +.SH "NAME" +\fBxmap\fR \- The Fast Internet Scanner +. +.SH "SYNOPSIS" +xmap [ \-4 | \-6 ] [ \-x ] [ \-p ] [ \-o ] [ OPTIONS\.\.\. ] [ ip|domain|range ] +. +.SH "DESCRIPTION" +\fIXMap\fR is a network tool for scanning any IPv6 & IPv4 address space (or large samples), reimplemented and improved thoroughly from ZMap\. XMap is capable of scanning the 32\-bits network space in around 45 minutes on a gigabit network connection, reaching ~98% theoretical line speed\. +. +.SH "OPTIONS" +. +.SS "BASIC OPTIONS" +. +.TP +\fB\-6\fR, \fB\-\-ipv6\fR +Scanning the IPv6 networks (default)\. +. +.TP +\fB\-4\fR, \fB\-\-ipv4\fR +Scanning the IPv4 networks\. +. +.TP +\fB\-x\fR, \fB\-\-max\-len=len\fR +Max IP bit length to scan (default = \fB32\fR)\. +. +.TP +\fBip\fR|\fBdomain\fR|\fBrange\fR +IP addresses or DNS hostnames to scan\. Accept IP ranges in CIDR block notation\. Max length of domains is 256, e\.g, 2001::/64, 192\.168\.0\.1/16, and www\.qq\.com/32\. Default to \fB::/0\fR and \fB0\.0\.0\.0/0\fR\. +. +.TP +\fB\-p\fR, \fB\-\-target\-port=port|range\fR +TCP or UDP port(s) number to scan (for SYN scans and basic UDP scans)\. Accepts port ranges with \fB,\fR and \fB\-\fR, e\.g\., \fB80,443,8080\-8081\fR\. With \fB\-\-target\-port\fR, one target is a \fB\fR\. +. +.TP +\fB\-o\fR, \fB\-\-output\-file=name\fR +When using an output module that uses a file, write results to this file\. Use \fB\-\fR for stdout\. +. +.TP +\fB\-b\fR, \fB\-\-blacklist\-file=path\fR +File of subnets to exclude, accept DNS hostnames, in CIDR notation, one\-per line\. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special\-purpose addresses\. An example blacklist file \fBblacklist4\.conf\fR for this purpose\. +. +.TP +\fB\-w\fR, \fB\-\-whitelist\-file=path\fR +File of subnets to include, accept DNS hostnames, in CIDR notation, one\-per line\. Specifying a whitelist file is equivalent to specifying to ranges directly on the command line interface, but allows specifying a large number of subnets\. \fBNote\fR: if you are specifying a large number of individual IP addresses (more than 1 million), you should instead use \fB\-\-list\-of\-ips\-file\fR\. An example whitelist file \fBwhitelist6\.conf\fR for this purpose\. +. +.TP +\fB\-I\fR, \fB\-\-list\-of\-ips\-file=path\fR +File of individual IP addresses to scan, one\-per line\. This feature allows you to scan a large number of unrelated addresses\. If you have a small number of IPs, it is faster to specify these on the command line or by using \fB\-\-whitelist\-file\fR\. \fBNote\fR: this should only be used when scanning more than 1 million addresses\. When used in with \fB\-\-whitelist\-file\fR, only hosts in the intersection of both sets will be scanned\. Hosts specified here, but included in the \fB\-\-blacklist\-file\fR will be excluded\. +. +.SS "SCAN OPTIONS" +. +.TP +\fB\-R\fR, \fB\-\-rate=pps\fR +Set the send rate in pkts/sec\. Note: when combined with \fB\-\-probes\fR or \fB\-\-retries\fR, this is total packets per second, not target number per second\. Setting the rate to \fB0\fR will scan at full line rate (no sleep)\. Default to \fB1\fR pps\. +. +.TP +\fB\-B\fR, \fB\-\-bandwidth=bps\fR +Set the send rate in bits/sec (supports suffixes G/g, M/m, and K/k, e\.g\. \-B 10M for 10 mbps)\. This overrides the \fB\-\-rate\fR flag\. Default to \fB0\fR bps\. +. +.TP +\fB\-\-batch=num\fR +Number of packets to send in a burst between checks to the ratelimit\. A batch size above 1 allows the sleep\-based rate\-limiter to be used with proportionally higher rates\. This can reduce CPU usage, in exchange for a bursty send rate (default = \fB1\fR)\. +. +.TP +\fB\-\-probes=num\fR +Number of probes to send to each target (default = \fB1\fR)\. +. +.TP +\fB\-\-retries=num\fR +Number of times to try resending a packet if the sendto call fails (default = \fB1\fR)\. +. +.TP +\fB\-n\fR, \fB\-\-max\-targets=num\fR +Capture number of targets to probe (default = \fB\-1\fR)\. +. +.TP +\fB\-k\fR, \fB\-\-max\-packets=num\fR +Capture number of packets to send (default = \fB\-1\fR)\. +. +.TP +\fB\-t\fR, \fB\-\-max\-runtime=secs\fR +Capture length of time for sending packets (default = \fB\-1\fR)\. +. +.TP +\fB\-N\fR, \fB\-\-max\-results=num\fR +Exit after receiving this many results (default = \fB\-1\fR)\. +. +.TP +\fB\-E\fR, \fB\-\-est\-elements=num\fR +Estimated number of results for unique (default = \fB5e8\fR)\. \fBNote\fR: XMap uses the bloomfilter to check the duplicate results, which costs some of the memory\. Choose the proper \fB\-\-est\-elements\fR to adapt to your memory capacity\. +. +.TP +\fB\-c\fR, \fB\-\-cooldown\-secs=secs\fR +How long to continue receiving after sending has completed (default = \fB5\fR)\. +. +.TP +\fB\-e\fR, \fB\-\-seed=num\fR +Seed used to select address permutation\. Use this if you want to scan addresses in the same order for multiple XMap runs (default = \fB0\fR)\. +. +.TP +\fB\-\-shards=num\fR +Split the scan up into N shards/partitions among different instances of xmap (default = \fB1\fR)\. When sharding, \fB\-\-seed\fR is required\. +. +.TP +\fB\-\-shard=num\fR +Set which shard to scan (default = \fB0\fR)\. Shards are 0\-indexed in the range [0, N), where N is the total number of shards\. When sharding\fB\-\-seed\fR is required\. +. +.SS "NETWORK OPTIONS" +. +.TP +\fB\-s\fR, \fB\-\-source\-port=port|range\fR +Source port(s) to send packets from\. Accept port ranges with \fB\-\fR, e\.g\., \fB12345\-54321\fR\. Default to \fB32768\-61000\fR\. +. +.TP +\fB\-S\fR, \fB\-\-source\-ip=ip|range\fR +Source address(es) to send packets from\. Either single IP or range\. Accept ip ranges with \fB,\fR and \fB\-\fR, e\.g\., 2001::1, 2001::2\-2001::10\. +. +.TP +\fB\-G\fR, \fB\-\-gateway\-mac=mac\fR +Gateway MAC address to send packets to (in case auto\-detection fails)\. +. +.TP +\fB\-\-source\-mac=mac\fR +Source MAC address to send packets from (in case auto\-detection fails)\. +. +.TP +\fB\-i\fR, \fB\-\-interface=name\fR +Network interface to use\. +. +.TP +\fB\-X\fR, \fB\-\-iplayer\fR +Send IP layer packets instead of ethernet packets (for non\-Ethernet interface)\. +. +.SS "PROBE OPTIONS" +XMap allows users to specify and write their own probe modules\. Probe modules are responsible for generating probe packets to send, and processing responses from hosts\. +. +.TP +\fB\-\-list\-probe\-modules\fR +List available probe modules (e\.g\., tcp_syn)\. +. +.TP +\fB\-M\fR, \fB\-\-probe\-module=name\fR +Select probe module (default = \fBicmp_echo\fR)\. +. +.TP +\fB\-\-probe\-args=args\fR +Arguments to pass to probe module\. +. +.TP +\fB\-\-probe\-ttl=hops\fR +Set TTL value for probe IP packets (default = \fB255\fR)\. +. +.TP +\fB\-\-list\-output\-fields\fR +List the fields the selected probe module can send to the output module\. +. +.SS "OUTPUT OPTIONS" +XMap allows users to specify and write their own output modules for use with XMap\. Output modules are responsible for processing the fieldsets returned by the probe module, and outputting them to the user\. Users can specify output fields, and write filters over the output fields\. +. +.TP +\fB\-\-list\-output\-modules\fR +List available output modules (e\.g\., csv)\. +. +.TP +\fB\-O\fR, \fB\-\-output\-module=name\fR +Select output module (default = \fBcsv\fR)\. +. +.TP +\fB\-\-output\-args=args\fR +Arguments to pass to output module\. +. +.TP +\fB\-f\fR, \fB\-\-output\-fields=fields\fR +Comma\-separated list of fields to output\. Accept fields with \fB,\fR and \fB*\fR\. +. +.TP +\fB\-\-output\-filter\fR +Specify an output filter over the fields defined by the probe module\. See the output filter section for more details\. +. +.SS "IID OPTIONS" +XMap allows users to specify and write their own iid modules for use with XMap\. IID modules are responsible for filling the left bits behind the probed prefix, and creating a whole target address\. +. +.P +processing the fieldsets returned by the probe module, and outputting them to the user\. Users can specify output fields, and write filters over the output fields\. +. +.TP +\fB\-\-list\-iid\-modules\fR +List available iid modules (e\.g\., low)\. +. +.TP +\fB\-U\fR, \fB\-\-iid\-module=name\fR +Select iid module (default = \fBlow\fR)\. +. +.TP +\fB\-\-iid\-args=args\fR +Arguments to pass to iid module\. +. +.TP +\fB\-\-iid\-num=num\fR +Number of iid for one target prefix\. +. +.SS "LOGGING AND METADATA OPTIONS" +. +.TP +\fB\-q\fR, \fB\-\-quiet\fR +Do not print status updates once per second\. +. +.TP +\fB\-v\fR, \fB\-\-verbosity=n\fR +Level of log detail (0\-5, default = \fB3\fR)\. +. +.TP +\fB\-l\fR, \fB\-\-log\-file=filename\fR +Output file for log messages\. By default, \fBstderr\fR\. +. +.TP +\fB\-L\fR, \fB\-\-log\-directory=path\fR +Write log entries to a timestamped file in this directory\. +. +.TP +\fB\-m\fR, \fB\-\-metadata\-file=filename\fR +Output file for scan metadata (JSON)\. +. +.TP +\fB\-u\fR, \fB\-\-status\-updates\-file\fR +Write scan progress updates to CSV file\. +. +.TP +\fB\-\-disable\-syslog\fR +Disables logging messages to syslog\. +. +.TP +\fB\-\-notes=notes\fR +Inject user\-specified notes into scan metadata\. +. +.TP +\fB\-\-user\-metadata=json\fR +Inject user\-specified JSON metadata into scan metadata\. +. +.SS "ADDITIONAL OPTIONS" +. +.TP +\fB\-T\fR, \fB\-\-sender\-threads=num\fR +Threads used to send packets\. XMap will attempt to detect the optimal number of send threads based on the number of processor cores\. +. +.TP +\fB\-C\fR, \fB\-\-config=filename\fR +Read a configuration file, which can specify any other options\. +. +.TP +\fB\-d\fR, \fB\-\-dryrun\fR +Print out each packet to stdout instead of sending it (useful for debugging)\. +. +.TP +\fB\-\-max\-sendto\-failures=num\fR +Maximum NIC sendto failures before scan is aborted\. +. +.TP +\fB\-\-min\-hitrate=rate\fR +Minimum hitrate that scan can hit before scan is aborted\. +. +.TP +\fB\-\-cores\fR +Comma\-separated list of cores to pin to\. +. +.TP +\fB\-\-ignore\-blacklist\-error\fR +Ignore invalid, malformed, or unresolvable entries in \fB\-\-whitelist\-file\fR and \fB\-\-blacklist\-file\fR\. +. +.TP +\fB\-\-ignore\-filelist\-error\fR +Ignore invalid, malformed, or unresolvable entries in \fB\-\-list\-of\-ips\-file\fR\. +. +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help and exit\. +. +.TP +\fB\-V\fR, \fB\-\-version\fR +Print version and exit\. +. +.SS "OUTPUT FILTERS" +Results generated by a probe module can be filtered before being passed to the output module\. Filters are defined over the output fields of a probe module\. Filters are written in a simple filtering language, similar to SQL, and are passed to XMap using the \fB\-\-output\-filter\fR option\. Output filters are commonly used to filter out duplicate results, or to only pass only successful responses to the output module\. +. +.P +Filter expressions are of the form \fB \fR\. The type of \fB\fR must be either a string or unsigned integer literal, and match the type of \fB\fR\. The valid operations for integer comparisons are \fB=\fR, \fB!=\fR, \fB<\fR, \fB>\fR, \fB<=\fR, \fB>=\fR\. The operations for string comparisons are \fB=\fR, \fB!=\fR\. The\fB\-\-list\-output\-fields\fR flag will print what fields and types are available for the selected probe module, and then exit\. +. +.P +Compound filter expressions may be constructed by combining filter expressions using parenthesis to specify order of operations, the \fB&&\fR (logical AND) and \fB||\fR (logical OR) operators\. +. +.P +For example, a filter for only successful, non\-duplicate responses would be written as: \fB\-\-output\-filter="success = 1 && repeat = 0"\fR\. +. +.SS "UDP PROBE MODULE OPTIONS" +These arguments are all passed using the \fB\-\-probe\-args=args\fR option\. Only one argument may be passed at a time\. +. +.TP +\fBfile:/path/to/file\fR +Path to payload file to send to each host over UDP\. +. +.TP +\fBtext:\fR +ASCII text to send to each destination host\. +. +.TP +\fBhex:\fR +Hex\-encoded binary to send to each destination host\. +. +.TP +\fBdir:/directory/to/file\fR +Directory to payload file to send to each host over UDP when probing multiple ports\. File extension priority: \fBpkt\fR>\fBtxt\fR>\fBhex\fR\. Each file is named by the port number, e\.g\., 53\.pkt for DNS payload\. +. +.TP +\fBtemplate:/path/to/template\fR +Path to template file\. For each destination host, the template file is populated, set as the UDP payload, and sent\. +. +.TP +\fBtemplate\-fields\fR +Print information about the allowed template fields and exit\. +. +.TP +\fBicmp\-type\-code\-str\fR +Print value of the icmp related filters and exit\. +. +.SS "MID\-SCAN CHANGES" +You can change the rate at which XMap is scanning mid\-scan by sending SIGUSR1 (increase) and SIGUSR2 (decrease) signals to XMap\. These will result in the scan rate increasing or decreasing by 5%\. +. +.SH "Examples" +. +.nf + +xmap + scan the ::/0\-32 space by Echo ping and output to stdout +xmap \-4 + scan the 0\.0\.0\.0/0\-32 space by Echo ping and output to stdout +xmap \-N 5 \-B 10M + find 5 alive IPv6 hosts, scanning at 10 Mb/s +xmap 2001::/8 2002::/16 + scan both subnets for 2001::/8\-32 and 2002::/16\-32 space +xmap \-x 64 2001::/32 \-U rand + scan 2001::/32\-64 with random IID, e\.g\., 2001::1783:ab42:9247:cb38 +xmap \-M icmp_echo \-O csv \-U low \-h + show help text for modules icmp_echo, csv, and low +xmap \-M tcp_syn \-p 80,443,8080\-8081 + scan the ::/0\-32 space for port 80,443,8080,8081 by TCP SYN ping +. +.fi + diff --git a/src/xmap.1.html b/src/xmap.1.html new file mode 100644 index 0000000..3392bce --- /dev/null +++ b/src/xmap.1.html @@ -0,0 +1,259 @@ + + + + + + xmap(1) - The Fast Internet Scanner + + + + +
+ + + +
    +
  1. xmap(1)
    2. +
  2. +
  3. xmap(1)
    4. +
+ +

NAME

+

+ xmap - The Fast Internet Scanner +

+ +

SYNOPSIS

+ +

xmap [ -4 | -6 ] [ -x <len> ] [ -p <port> ] [ -o <outfile> ] [ OPTIONS... ] [ ip|domain|range ]

+ +

DESCRIPTION

+ +

XMap is a network tool for scanning any IPv6 & IPv4 address space (or large samples), reimplemented and improved thoroughly from ZMap. XMap is capable of scanning the 32-bits network space in around 45 minutes on a gigabit network connection, reaching ~98% theoretical line speed.

+ +

OPTIONS

+ +

BASIC OPTIONS

+ +
+
-6, --ipv6

Scanning the IPv6 networks (default).

+
-4, --ipv4

Scanning the IPv4 networks.

+
-x, --max-len=len

Max IP bit length to scan (default = 32).

+
ip|domain|range

IP addresses or DNS hostnames to scan. Accept IP ranges in CIDR block notation. Max length of domains is 256, e.g, 2001::/64, 192.168.0.1/16, and www.qq.com/32. Default to ::/0 and 0.0.0.0/0.

+
-p, --target-port=port|range

TCP or UDP port(s) number to scan (for SYN scans and basic UDP scans). Accepts port ranges with , and -, e.g., 80,443,8080-8081. With --target-port, one target is a <ip/x, port>.

+
-o, --output-file=name

When using an output module that uses a file, write results to this file. Use - for stdout.

+
-b, --blacklist-file=path

File of subnets to exclude, accept DNS hostnames, in CIDR notation, one-per line. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special-purpose addresses. An example blacklist file blacklist4.conf for this purpose.

+
-w, --whitelist-file=path

File of subnets to include, accept DNS hostnames, in CIDR notation, one-per line. Specifying a whitelist file is equivalent to specifying to ranges directly on the command line interface, but allows specifying a large number of subnets. Note: if you are specifying a large number of individual IP addresses (more than 1 million), you should instead use --list-of-ips-file. An example whitelist file whitelist6.conf for this purpose.

+
-I, --list-of-ips-file=path

File of individual IP addresses to scan, one-per line. This feature allows you to scan a large number of unrelated addresses. If you have a small number of IPs, it is faster to specify these on the command line or by using --whitelist-file. Note: this should only be used when scanning more than 1 million addresses. When used in with --whitelist-file, only hosts in the intersection of both sets will be scanned. Hosts specified here, but included in the --blacklist-file will be excluded.

+
+ + +

SCAN OPTIONS

+ +
+
-R, --rate=pps

Set the send rate in pkts/sec. Note: when combined with --probes or --retries, this is total packets per second, not target number per second. Setting the rate to 0 will scan at full line rate (no sleep). Default to 1 pps.

+
-B, --bandwidth=bps

Set the send rate in bits/sec (supports suffixes G/g, M/m, and K/k, e.g. -B 10M for 10 mbps). This overrides the --rate flag. Default to 0 bps.

+
--batch=num

Number of packets to send in a burst between checks to the ratelimit. A batch size above 1 allows the sleep-based rate-limiter to be used with proportionally higher rates. This can reduce CPU usage, in exchange for a bursty send rate (default = 1).

+
--probes=num

Number of probes to send to each target (default = 1).

+
--retries=num

Number of times to try resending a packet if the sendto call fails (default = 1).

+
-n, --max-targets=num

Capture number of targets to probe (default = -1).

+
-k, --max-packets=num

Capture number of packets to send (default = -1).

+
-t, --max-runtime=secs

Capture length of time for sending packets (default = -1).

+
-N, --max-results=num

Exit after receiving this many results (default = -1).

+
-E, --est-elements=num

Estimated number of results for unique (default = 5e8). Note: XMap uses the bloomfilter to check the duplicate results, which costs some of the memory. Choose the proper --est-elements to adapt to your memory capacity.

+
-c, --cooldown-secs=secs

How long to continue receiving after sending has completed (default = 5).

+
-e, --seed=num

Seed used to select address permutation. Use this if you want to scan addresses in the same order for multiple XMap runs (default = 0).

+
--shards=num

Split the scan up into N shards/partitions among different instances of xmap (default = 1). When sharding, --seed is required.

+
--shard=num

Set which shard to scan (default = 0). Shards are 0-indexed in the range [0, N), where N is the total number of shards. When sharding--seed is required.

+
+ + +

NETWORK OPTIONS

+ +
+
-s, --source-port=port|range

Source port(s) to send packets from. Accept port ranges with -, e.g., 12345-54321. Default to 32768-61000.

+
-S, --source-ip=ip|range

Source address(es) to send packets from. Either single IP or range. Accept ip ranges with , and -, e.g., 2001::1, 2001::2-2001::10.

+
-G, --gateway-mac=mac

Gateway MAC address to send packets to (in case auto-detection fails).

+
--source-mac=mac

Source MAC address to send packets from (in case auto-detection fails).

+
-i, --interface=name

Network interface to use.

+
-X, --iplayer

Send IP layer packets instead of ethernet packets (for non-Ethernet interface).

+
+ + +

PROBE OPTIONS

+ +

XMap allows users to specify and write their own probe modules. Probe modules are responsible for generating probe packets to send, and processing responses from hosts.

+ +
+
--list-probe-modules

List available probe modules (e.g., tcp_syn).

+
-M, --probe-module=name

Select probe module (default = icmp_echo).

+
--probe-args=args

Arguments to pass to probe module.

+
--probe-ttl=hops

Set TTL value for probe IP packets (default = 255).

+
--list-output-fields

List the fields the selected probe module can send to the output module.

+
+ + +

OUTPUT OPTIONS

+ +

XMap allows users to specify and write their own output modules for use with XMap. Output modules are responsible for processing the fieldsets returned by the probe module, and outputting them to the user. Users can specify output fields, and write filters over the output fields.

+ +
+
--list-output-modules
List available output modules (e.g., csv).
+
-O, --output-module=name
Select output module (default = csv).
+
--output-args=args
Arguments to pass to output module.
+
-f, --output-fields=fields
Comma-separated list of fields to output. Accept fields with , and *.
+
--output-filter
Specify an output filter over the fields defined by the probe module. See the output filter section for more details.
+
+ + +

IID OPTIONS

+ +

XMap allows users to specify and write their own iid modules for use with XMap. IID modules are responsible for filling the left bits behind the probed prefix, and creating a whole target address.

+ +

processing the fieldsets returned by the probe module, and outputting them to the user. Users can specify output fields, and write filters over the output fields.

+ +
+
--list-iid-modules

List available iid modules (e.g., low).

+
-U, --iid-module=name

Select iid module (default = low).

+
--iid-args=args

Arguments to pass to iid module.

+
--iid-num=num

Number of iid for one target prefix.

+
+ + +

LOGGING AND METADATA OPTIONS

+ +
+
-q, --quiet

Do not print status updates once per second.

+
-v, --verbosity=n

Level of log detail (0-5, default = 3).

+
-l, --log-file=filename

Output file for log messages. By default, stderr.

+
-L, --log-directory=path

Write log entries to a timestamped file in this directory.

+
-m, --metadata-file=filename

Output file for scan metadata (JSON).

+
-u, --status-updates-file

Write scan progress updates to CSV file.

+
--disable-syslog

Disables logging messages to syslog.

+
--notes=notes

Inject user-specified notes into scan metadata.

+
--user-metadata=json

Inject user-specified JSON metadata into scan metadata.

+
+ + +

ADDITIONAL OPTIONS

+ +
+
-T, --sender-threads=num

Threads used to send packets. XMap will attempt to detect the optimal number of send threads based on the number of processor cores.

+
-C, --config=filename

Read a configuration file, which can specify any other options.

+
-d, --dryrun

Print out each packet to stdout instead of sending it (useful for debugging).

+
--max-sendto-failures=num

Maximum NIC sendto failures before scan is aborted.

+
--min-hitrate=rate

Minimum hitrate that scan can hit before scan is aborted.

+
--cores

Comma-separated list of cores to pin to.

+
--ignore-blacklist-error

Ignore invalid, malformed, or unresolvable entries in --whitelist-file and --blacklist-file.

+
--ignore-filelist-error

Ignore invalid, malformed, or unresolvable entries in --list-of-ips-file.

+
-h, --help

Print help and exit.

+
-V, --version

Print version and exit.

+
+ + +

OUTPUT FILTERS

+ +

Results generated by a probe module can be filtered before being passed to the output module. Filters are defined over the output fields of a probe module. Filters are written in a simple filtering language, similar to SQL, and are passed to XMap using the --output-filter option. Output filters are commonly used to filter out duplicate results, or to only pass only successful responses to the output module.

+ +

Filter expressions are of the form <fieldname> <operation> <value>. The type of <value> must be either a string or unsigned integer literal, and match the type of <fieldname>. The valid operations for integer comparisons are =, !=, <, >, <=, >=. The operations for string comparisons are =, !=. The--list-output-fields flag will print what fields and types are available for the selected probe module, and then exit.

+ +

Compound filter expressions may be constructed by combining filter expressions using parenthesis to specify order of operations, the && (logical AND) and || (logical OR) operators.

+ +

For example, a filter for only successful, non-duplicate responses would be written as: --output-filter="success = 1 && repeat = 0".

+ +

UDP PROBE MODULE OPTIONS

+ +

These arguments are all passed using the --probe-args=args option. Only one argument may be passed at a time.

+ +
+
file:/path/to/file

Path to payload file to send to each host over UDP.

+
text:<text>

ASCII text to send to each destination host.

+
hex:<hex>

Hex-encoded binary to send to each destination host.

+
dir:/directory/to/file

Directory to payload file to send to each host over UDP when probing multiple ports. + File extension priority: pkt>txt>hex. Each file is named by the port number, e.g., 53.pkt for DNS payload.

+
template:/path/to/template

Path to template file. For each destination host, the template file is populated, set as the UDP payload, and sent.

+
template-fields

Print information about the allowed template fields and exit.

+
icmp-type-code-str

Print value of the icmp related filters and exit.

+
+ + +

MID-SCAN CHANGES

+ +

You can change the rate at which XMap is scanning mid-scan by sending SIGUSR1 (increase) and SIGUSR2 (decrease) signals to XMap. These will result in the scan rate increasing or decreasing by 5%.

+ +

Examples

+ +
xmap
+    scan the ::/0-32 space by Echo ping and output to stdout
+xmap -4
+    scan the 0.0.0.0/0-32 space by Echo ping and output to stdout
+xmap -N 5 -B 10M
+    find 5 alive IPv6 hosts, scanning at 10 Mb/s
+xmap 2001::/8 2002::/16
+    scan both subnets for 2001::/8-32 and 2002::/16-32 space
+xmap -x 64 2001::/32 -U rand
+    scan 2001::/32-64 with random IID, e.g., 2001::1783:ab42:9247:cb38
+xmap -M icmp_echo -O csv -U low -h
+    show help text for modules icmp_echo, csv, and low
+xmap -M tcp_syn -p 80,443,8080-8081
+    scan the ::/0-32 space for port 80,443,8080,8081 by TCP SYN ping        
+
+ + +
    +
  1. +
  2. April 2021
    3. +
  3. xmap(1)
    4. +
+ +
+ + diff --git a/src/xmap.1.ronn b/src/xmap.1.ronn new file mode 100644 index 0000000..c4aaeff --- /dev/null +++ b/src/xmap.1.ronn @@ -0,0 +1,279 @@ +xmap(1) - The Fast Internet Scanner +=================================== + +## SYNOPSIS + +xmap [ -4 | -6 ] [ -x <len> ] [ -p <port> ] [ -o <outfile> ] [ OPTIONS... ] [ ip|domain|range ] + +## DESCRIPTION + +*XMap* is a network tool for scanning any IPv6 & IPv4 address space (or large samples), reimplemented and improved thoroughly from ZMap. XMap is capable of scanning the 32-bits network space in around 45 minutes on a gigabit network connection, reaching ~98% theoretical line speed. + +## OPTIONS + +### BASIC OPTIONS ### + + * `-6`, `--ipv6`: + Scanning the IPv6 networks (default). + + * `-4`, `--ipv4`: + Scanning the IPv4 networks. + + * `-x`, `--max-len=len`: + Max IP bit length to scan (default = `32`). + + * `ip`|`domain`|`range`: + IP addresses or DNS hostnames to scan. Accept IP ranges in CIDR block notation. Max length of domains is 256, e.g, 2001::/64, 192.168.0.1/16, and www.qq.com/32. Default to `::/0` and `0.0.0.0/0`. + + * `-p`, `--target-port=port|range`: + TCP or UDP port(s) number to scan (for SYN scans and basic UDP scans). Accepts port ranges with `,` and `-`, e.g., `80,443,8080-8081`. With `--target-port`, one target is a ****. + + * `-o`, `--output-file=name`: + When using an output module that uses a file, write results to this file. Use `-` for stdout. + + * `-b`, `--blacklist-file=path`: + File of subnets to exclude, accept DNS hostnames, in CIDR notation, one-per line. It is recommended you use this to exclude RFC 1918 addresses, multicast, IANA reserved space, and other IANA special-purpose addresses. An example blacklist file **blacklist4.conf** for this purpose. + + * `-w`, `--whitelist-file=path`: + File of subnets to include, accept DNS hostnames, in CIDR notation, one-per line. Specifying a whitelist file is equivalent to specifying to ranges directly on the command line interface, but allows specifying a large number of subnets. **Note**: if you are specifying a large number of individual IP addresses (more than 1 million), you should instead use `--list-of-ips-file`. An example whitelist file **whitelist6.conf** for this purpose. + + * `-I`, `--list-of-ips-file=path`: + File of individual IP addresses to scan, one-per line. This feature allows you to scan a large number of unrelated addresses. If you have a small number of IPs, it is faster to specify these on the command line or by using `--whitelist-file`. **Note**: this should only be used when scanning more than 1 million addresses. When used in with `--whitelist-file`, only hosts in the intersection of both sets will be scanned. Hosts specified here, but included in the `--blacklist-file` will be excluded. + +### SCAN OPTIONS ### + + * `-R`, `--rate=pps`: + Set the send rate in pkts/sec. Note: when combined with `--probes` or `--retries`, this is total packets per second, not target number per second. Setting the rate to `0` will scan at full line rate (no sleep). Default to `1` pps. + + * `-B`, `--bandwidth=bps`: + Set the send rate in bits/sec (supports suffixes G/g, M/m, and K/k, e.g. -B 10M for 10 mbps). This overrides the `--rate` flag. Default to `0` bps. + + * `--batch=num`: + Number of packets to send in a burst between checks to the ratelimit. A batch size above 1 allows the sleep-based rate-limiter to be used with proportionally higher rates. This can reduce CPU usage, in exchange for a bursty send rate (default = `1`). + + * `--probes=num`: + Number of probes to send to each target (default = `1`). + + * `--retries=num`: + Number of times to try resending a packet if the sendto call fails (default = `1`). + + * `-n`, `--max-targets=num`: + Capture number of targets to probe (default = `-1`). + + * `-k`, `--max-packets=num`: + Capture number of packets to send (default = `-1`). + + * `-t`, `--max-runtime=secs`: + Capture length of time for sending packets (default = `-1`). + + * `-N`, `--max-results=num`: + Exit after receiving this many results (default = `-1`). + + * `-E`, `--est-elements=num`: + Estimated number of results for unique (default = `5e8`). **Note**: XMap uses the bloomfilter to check the duplicate results, which costs some of the memory. Choose the proper `--est-elements` to adapt to your memory capacity. + + * `-c`, `--cooldown-secs=secs`: + How long to continue receiving after sending has completed (default = `5`). + + * `-e`, `--seed=num`: + Seed used to select address permutation. Use this if you want to scan addresses in the same order for multiple XMap runs (default = `0`). + + * `--shards=num`: + Split the scan up into N shards/partitions among different instances of xmap (default = `1`). When sharding, `--seed` is required. + + * `--shard=num`: + Set which shard to scan (default = `0`). Shards are 0-indexed in the range [0, N), where N is the total number of shards. When sharding`--seed` is required. + + +### NETWORK OPTIONS ### + + * `-s`, `--source-port=port|range`: + Source port(s) to send packets from. Accept port ranges with `-`, e.g., `12345-54321`. Default to `32768-61000`. + + * `-S`, `--source-ip=ip|range`: + Source address(es) to send packets from. Either single IP or range. Accept ip ranges with `,` and `-`, e.g., 2001::1, 2001::2-2001::10. + + * `-G`, `--gateway-mac=mac`: + Gateway MAC address to send packets to (in case auto-detection fails). + + * `--source-mac=mac`: + Source MAC address to send packets from (in case auto-detection fails). + + * `-i`, `--interface=name`: + Network interface to use. + + * `-X`, `--iplayer`: + Send IP layer packets instead of ethernet packets (for non-Ethernet interface). + +### PROBE OPTIONS ### + +XMap allows users to specify and write their own probe modules. Probe modules are responsible for generating probe packets to send, and processing responses from hosts. + + * `--list-probe-modules`: + List available probe modules (e.g., tcp_syn). + + * `-M`, `--probe-module=name`: + Select probe module (default = `icmp_echo`). + + * `--probe-args=args`: + Arguments to pass to probe module. + + * `--probe-ttl=hops`: + Set TTL value for probe IP packets (default = `255`). + + * `--list-output-fields`: + List the fields the selected probe module can send to the output module. + +### OUTPUT OPTIONS ### + +XMap allows users to specify and write their own output modules for use with XMap. Output modules are responsible for processing the fieldsets returned by the probe module, and outputting them to the user. Users can specify output fields, and write filters over the output fields. + + * `--list-output-modules`: + List available output modules (e.g., csv). + * `-O`, `--output-module=name`: + Select output module (default = `csv`). + * `--output-args=args`: + Arguments to pass to output module. + * `-f`, `--output-fields=fields`: + Comma-separated list of fields to output. Accept fields with `,` and `*`. + * `--output-filter`: + Specify an output filter over the fields defined by the probe module. See the output filter section for more details. + +### IID OPTIONS ### + +XMap allows users to specify and write their own iid modules for use with XMap. IID modules are responsible for filling the left bits behind the probed prefix, and creating a whole target address. + +processing the fieldsets returned by the probe module, and outputting them to the user. Users can specify output fields, and write filters over the output fields. + + * `--list-iid-modules`: + List available iid modules (e.g., low). + + * `-U`, `--iid-module=name`: + Select iid module (default = `low`). + + * `--iid-args=args`: + Arguments to pass to iid module. + + * `--iid-num=num`: + Number of iid for one target prefix. + +### LOGGING AND METADATA OPTIONS ### + + * `-q`, `--quiet`: + Do not print status updates once per second. + + * `-v`, `--verbosity=n`: + Level of log detail (0-5, default = `3`). + + * `-l`, `--log-file=filename`: + Output file for log messages. By default, `stderr`. + + * `-L`, `--log-directory=path`: + Write log entries to a timestamped file in this directory. + + * `-m`, `--metadata-file=filename`: + Output file for scan metadata (JSON). + + * `-u`, `--status-updates-file`: + Write scan progress updates to CSV file. + + * `--disable-syslog`: + Disables logging messages to syslog. + + * `--notes=notes`: + Inject user-specified notes into scan metadata. + + * `--user-metadata=json`: + Inject user-specified JSON metadata into scan metadata. + +### ADDITIONAL OPTIONS ### + + * `-T`, `--sender-threads=num`: + Threads used to send packets. XMap will attempt to detect the optimal number of send threads based on the number of processor cores. + + * `-C`, `--config=filename`: + Read a configuration file, which can specify any other options. + + * `-d`, `--dryrun`: + Print out each packet to stdout instead of sending it (useful for debugging). + + * `--max-sendto-failures=num`: + Maximum NIC sendto failures before scan is aborted. + + * `--min-hitrate=rate`: + Minimum hitrate that scan can hit before scan is aborted. + + * `--cores`: + Comma-separated list of cores to pin to. + + * `--ignore-blacklist-error`: + Ignore invalid, malformed, or unresolvable entries in `--whitelist-file` and `--blacklist-file`. + + * `--ignore-filelist-error`: + Ignore invalid, malformed, or unresolvable entries in `--list-of-ips-file`. + + * `-h`, `--help`: + Print help and exit. + + * `-V`, `--version`: + Print version and exit. + +### OUTPUT FILTERS ### + +Results generated by a probe module can be filtered before being passed to the output module. Filters are defined over the output fields of a probe module. Filters are written in a simple filtering language, similar to SQL, and are passed to XMap using the `--output-filter` option. Output filters are commonly used to filter out duplicate results, or to only pass only successful responses to the output module. + +Filter expressions are of the form ` `. The type of `` must be either a string or unsigned integer literal, and match the type of ``. The valid operations for integer comparisons are `=`, `!=`, `<`, `>`, `<=`, `>=`. The operations for string comparisons are `=`, `!=`. The`--list-output-fields` flag will print what fields and types are available for the selected probe module, and then exit. + +Compound filter expressions may be constructed by combining filter expressions using parenthesis to specify order of operations, the `&&` (logical AND) and `||` (logical OR) operators. + +For example, a filter for only successful, non-duplicate responses would be written as: `--output-filter="success = 1 && repeat = 0"`. + +### UDP PROBE MODULE OPTIONS ### + +These arguments are all passed using the `--probe-args=args` option. Only one argument may be passed at a time. + + * `file:/path/to/file`: + Path to payload file to send to each host over UDP. + + * `text:`: + ASCII text to send to each destination host. + + * `hex:`: + Hex-encoded binary to send to each destination host. + + * `dir:/directory/to/file`: + Directory to payload file to send to each host over UDP when probing multiple ports. + File extension priority: `pkt`>`txt`>`hex`. Each file is named by the port number, e.g., 53.pkt for DNS payload. + + * `template:/path/to/template`: + Path to template file. For each destination host, the template file is populated, set as the UDP payload, and sent. + + * `template-fields`: + Print information about the allowed template fields and exit. + + * `icmp-type-code-str`: + Print value of the icmp related filters and exit. + +### MID-SCAN CHANGES ### + +You can change the rate at which XMap is scanning mid-scan by sending SIGUSR1 (increase) and SIGUSR2 (decrease) signals to XMap. These will result in the scan rate increasing or decreasing by 5%. + +## Examples + +``` +xmap + scan the ::/0-32 space by Echo ping and output to stdout +xmap -4 + scan the 0.0.0.0/0-32 space by Echo ping and output to stdout +xmap -N 5 -B 10M + find 5 alive IPv6 hosts, scanning at 10 Mb/s +xmap 2001::/8 2002::/16 + scan both subnets for 2001::/8-32 and 2002::/16-32 space +xmap -x 64 2001::/32 -U rand + scan 2001::/32-64 with random IID, e.g., 2001::1783:ab42:9247:cb38 +xmap -M icmp_echo -O csv -U low -h + show help text for modules icmp_echo, csv, and low +xmap -M tcp_syn -p 80,443,8080-8081 + scan the ::/0-32 space for port 80,443,8080,8081 by TCP SYN ping +``` + diff --git a/src/xmap.c b/src/xmap.c new file mode 100644 index 0000000..de662c5 --- /dev/null +++ b/src/xmap.c @@ -0,0 +1,1197 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "get_gateway.h" +#include "ip_target_file.h" +#include "monitor.h" +#include "recv.h" +#include "send.h" +#include "shard.h" +#include "socket.h" +#include "state.h" +#include "summary.h" +#include "utility.h" +#include "xopt.h" + +#include "iid_modules/iid_modules.h" +#include "output_modules/output_modules.h" +#include "probe_modules/probe_modules.h" + +#include "../lib/blocklist.h" +#include "../lib/gmp-ext.h" +#include "../lib/logger.h" +#include "../lib/random.h" +#include "../lib/util.h" +#include "../lib/xalloc.h" + +#ifdef PFRING +#include +static int32_t distrib_func(pfring_zc_pkt_buff *pkt, pfring_zc_queue *in_queue, + void *arg) { + (void) pkt; + (void) in_queue; + (void) arg; + + return 0; +} +#endif + +pthread_mutex_t recv_ready_mutex = PTHREAD_MUTEX_INITIALIZER; + +typedef struct send_args { + uint32_t cpu; + sock_t sock; + shard_t *shard; +} send_args_t; + +typedef struct recv_args { + uint32_t cpu; +} recv_args_t; + +typedef struct mon_start_args { + uint32_t cpu; + iterator_t * it; + pthread_mutex_t *recv_ready_mutex; +} mon_start_args_t; + +const char *default_help_text = + "By default, XMap prints out unique, successful IPv6/IPv4 addresses (e.g., " + "a Echo reply from a Echo scan) in ASCII form (e.g., 2001::1234, " + "192.168.1.5) to stdout (--output-file=-) or the specified output file " + "(--output-file=file). Internally this is handled by the `csv' output " + "module and is equivalent to running xmap --output-module=csv " + "--output-fields=\"saddr,\" --output-filter=\"success = 1 && repeat = 0\"."; + +static void *start_send(void *args) { + send_args_t *s = (send_args_t *) args; + log_debug("xmap", "pinning a send thread to core %u", s->cpu); + + set_cpu(s->cpu); + send_run(s->sock, s->shard); + free(s); + + return NULL; +} + +static void *start_recv(void *args) { + recv_args_t *r = (recv_args_t *) args; + log_debug("xmap", "pinning receive thread to core %u", r->cpu); + + set_cpu(r->cpu); + recv_run(&recv_ready_mutex); + + return NULL; +} + +static void *start_mon(void *args) { + mon_start_args_t *mon_arg = (mon_start_args_t *) args; + log_debug("xmap", "pinning monitor thread to core %u", mon_arg->cpu); + + set_cpu(mon_arg->cpu); + monitor_run(mon_arg->it, mon_arg->recv_ready_mutex); + free(mon_arg); + + return NULL; +} + +static void start_xmap(void) { + log_debug("xmap", "xmap start."); + + if (xconf.iface == NULL) { + xconf.iface = get_default_iface(); + if (!xconf.iface) { + log_fatal("xmap", + "could not detect default interface. Try specifying a " + "interface `-i|--interface='", + xconf.ipv46_flag, xconf.iface); + } + log_debug("xmap", + "no interface provided. will use default interface: %s.", + xconf.iface); + } + + if (xconf.number_source_ips == 0) { + if (get_iface_ip(xconf.iface, xconf.source_ip_addresses[0], + xconf.ipv46_flag)) { + log_fatal("xmap", + "could not detect default IPv%d address for %s. Try " + "specifying a source address `-S|--source-ip'", + xconf.ipv46_flag, xconf.iface); + } + xconf.number_source_ips++; + log_debug( + "xmap", + "no source IPv%d address given. will use default address: %s", + xconf.ipv46_flag, + inet_in2constr(xconf.source_ip_addresses[0], xconf.ipv46_flag)); + } + + // Get the source hardware address, and give it to the probe module + if (!xconf.hw_mac_set) { + memset(xconf.hw_mac, 0, MAC_ADDR_LEN); + if (get_iface_hw_addr(xconf.iface, xconf.hw_mac)) { + log_fatal("xmap", + "could not retrieve hardware address for interface: %s", + xconf.iface); + } + log_debug("xmap", + "no source MAC provided. Automatically detected " + "%02x:%02x:%02x:%02x:%02x:%02x for %s", + xconf.hw_mac[0], xconf.hw_mac[1], xconf.hw_mac[2], + xconf.hw_mac[3], xconf.hw_mac[4], xconf.hw_mac[5], + xconf.iface); + } + log_debug("xmap", "source MAC address %02x:%02x:%02x:%02x:%02x:%02x", + xconf.hw_mac[0], xconf.hw_mac[1], xconf.hw_mac[2], + xconf.hw_mac[3], xconf.hw_mac[4], xconf.hw_mac[5]); + + if (!xconf.gw_mac_set) { + memset(xconf.gw_ip, 0, xconf.ipv46_bytes); + if (get_default_gw_ip(xconf.gw_ip, xconf.iface)) { + log_fatal("xmap", + "could not detect default gateway address for %s. Try " + "setting default gateway mac address (-G).", + xconf.iface); + } + // TODO only support to get gw's IPv4 address: IPV4_FLAG + log_debug("xmap", "found gateway IPv4 address %s on %s", + inet_in2constr(xconf.gw_ip, IPV4_FLAG), xconf.iface); + + memset(&xconf.gw_mac, 0, MAC_ADDR_LEN); + if (get_hw_addr(xconf.gw_ip, xconf.iface, xconf.gw_mac)) { + log_fatal("xmap", + "could not detect GW MAC address for %s on %s. Try " + "setting default gateway mac address (-G), or run \"arp " + "\" in terminal.", + inet_in2constr(xconf.gw_ip, IPV4_FLAG), xconf.iface); + } + xconf.gw_mac_set = 1; + } + log_debug("xmap", "gateway MAC address %02x:%02x:%02x:%02x:%02x:%02x", + xconf.gw_mac[0], xconf.gw_mac[1], xconf.gw_mac[2], + xconf.gw_mac[3], xconf.gw_mac[4], xconf.gw_mac[5]); + + // PFRING +#ifdef PFRING +#define MAX_CARD_SLOTS 32768 +#define QUEUE_LEN 8192 +#define PMAP_PF_BUFFER_SIZE 1536 +#define PMAP_PF_ZC_CLUSTER_ID 9627 + uint32_t user_buffers = xconf.senders * 256; + uint32_t queue_buffers = xconf.senders * QUEUE_LEN; + uint32_t card_buffers = 2 * MAX_CARD_SLOTS; + uint32_t total_buffers = user_buffers + queue_buffers + card_buffers + 2; + uint32_t metadata_len = 0; + uint32_t numa_node = 0; // TODO + xconf.pf.cluster = pfring_zc_create_cluster( + PMAP_PF_ZC_CLUSTER_ID, PMAP_PF_BUFFER_SIZE, metadata_len, total_buffers, + numa_node, NULL, 0); + if (xconf.pf.cluster == NULL) { + log_fatal("xmap", "Could not create zc cluster: %s", strerror(errno)); + } + + xconf.pf.buffers = xcalloc(user_buffers, sizeof(pfring_zc_pkt_buff *)); + for (uint32_t i = 0; i < user_buffers; ++i) { + xconf.pf.buffers[i] = pfring_zc_get_packet_handle(xconf.pf.cluster); + if (xconf.pf.buffers[i] == NULL) { + log_fatal("xmap", "Could not get ZC packet handle"); + } + } + + xconf.pf.send = + pfring_zc_open_device(xconf.pf.cluster, xconf.iface, tx_only, 0); + if (xconf.pf.send == NULL) { + log_fatal("xmap", "Could not open device %s for TX. [%s]", xconf.iface, + strerror(errno)); + } + + xconf.pf.recv = + pfring_zc_open_device(xconf.pf.cluster, xconf.iface, rx_only, 0); + if (xconf.pf.recv == NULL) { + log_fatal("xmap", "Could not open device %s for RX. [%s]", xconf.iface, + strerror(errno)); + } + + xconf.pf.queues = xcalloc(xconf.senders, sizeof(pfring_zc_queue *)); + for (uint32_t i = 0; i < xconf.senders; ++i) { + xconf.pf.queues[i] = + pfring_zc_create_queue(xconf.pf.cluster, QUEUE_LEN); + if (xconf.pf.queues[i] == NULL) { + log_fatal("xmap", "Could not create queue: %s", strerror(errno)); + } + } + + xconf.pf.prefetches = pfring_zc_create_buffer_pool(xconf.pf.cluster, 8); + if (xconf.pf.prefetches == NULL) { + log_fatal("xmap", "Could not open prefetch pool: %s", strerror(errno)); + } +#endif + + // Initialization + log_info("xmap", "probe network: ipv%d", xconf.ipv46_flag); + log_info("xmap", "probe module: %s", xconf.probe_module->name); + log_info("xmap", "output module: %s", xconf.output_module->name); + log_info("xmap", "iid module: %s", xconf.iid_module->name); + + if (xconf.output_module && xconf.output_module->init) { + if (xconf.output_module->init(&xconf, xconf.output_fields, + xconf.output_fields_len)) { + log_fatal("xmap", "output module did not initialize successfully."); + } + } + + // Send global init + iterator_t *it = send_init(); + if (!it) log_fatal("xmap", "unable to initialize sending component"); + + if (xconf.output_module && xconf.output_module->start) + xconf.output_module->start(&xconf, &xsend, &xrecv); + + // start threads + uint32_t cpu = 0; + pthread_t *tsend, trecv, tmon; + + // recv thread + if (!xconf.dryrun) { // TODO how many target will reply? + if (bloom_filter_init(&xrecv.bf, xconf.est_elements, (float) 1e-5) == + BLOOM_FAILURE) { + log_fatal("xmap", + "unable to create bloomfilter for unique results"); + } + log_debug("xmap", "bloomfilter for unique results <= 4e9"); + + recv_args_t *recv_arg = xmalloc(sizeof(recv_args_t)); + recv_arg->cpu = xconf.pin_cores[cpu % xconf.pin_cores_len]; + cpu += 1; + int r = pthread_create(&trecv, NULL, start_recv, recv_arg); + if (r != 0) { + log_fatal("xmap", "unable to create recv thread"); + } + + for (;;) { + pthread_mutex_lock(&recv_ready_mutex); + if (xconf.recv_ready) { + pthread_mutex_unlock(&recv_ready_mutex); + break; + } + pthread_mutex_unlock(&recv_ready_mutex); + } + } + +#ifdef PFRING + pfring_zc_worker *zw = pfring_zc_run_balancer( + xconf.pf.queues, &xconf.pf.send, xconf.senders, 1, xconf.pf.prefetches, + round_robin_bursts_policy, NULL, distrib_func, NULL, 0, + xconf.pin_cores[cpu & xconf.pin_cores_len]); + cpu += 1; +#endif + + // send thread + tsend = xmalloc(xconf.senders * sizeof(pthread_t)); + for (uint8_t i = 0; i < xconf.senders; i++) { + sock_t sock; + if (xconf.dryrun) { + sock = get_dryrun_socket(); + } else { + sock = get_socket(i); + } + + send_args_t *arg = xmalloc(sizeof(send_args_t)); + arg->sock = sock; + arg->shard = get_shard(it, i); + arg->cpu = xconf.pin_cores[cpu % xconf.pin_cores_len]; + cpu += 1; + + int r = pthread_create(&tsend[i], NULL, start_send, arg); + if (r != 0) { + log_fatal("xmap", "unable to create send thread"); + } + } + log_debug("xmap", "%d sender threads spawned", xconf.senders); + + // monitor thread + if (!xconf.dryrun) { + monitor_init(); + mon_start_args_t *mon_arg = xmalloc(sizeof(mon_start_args_t)); + mon_arg->it = it; + mon_arg->recv_ready_mutex = &recv_ready_mutex; + mon_arg->cpu = xconf.pin_cores[cpu % xconf.pin_cores_len]; + + int r = pthread_create(&tmon, NULL, start_mon, mon_arg); + if (r != 0) { + log_fatal("xmap", "unable to create monitor thread"); + } + } + +#ifndef PFRING + drop_privs(); +#endif + + // wait for send completion + for (uint8_t i = 0; i < xconf.senders; i++) { + int r = pthread_join(tsend[i], NULL); + if (r != 0) { + log_fatal("xmap", "unable to join send thread"); + } + } + log_debug("xmap", "senders finished"); + +#ifdef PFRING + pfring_zc_kill_worker(zw); + pfring_zc_sync_queue(xconf.pf.send, tx_only); + log_debug("xmap", "send queue flushed"); +#endif + + // wait for recv & monitor completion + if (!xconf.dryrun) { + int r = pthread_join(trecv, NULL); + if (r != 0) { + log_fatal("xmap", "unable to join recv thread"); + } + + bloom_filter_destroy(&xrecv.bf); + log_debug("xmap", "receiver finished"); + + if (!xconf.quiet || xconf.status_updates_file) { + r = pthread_join(tmon, NULL); + if (r != 0) { + log_fatal("xmap", "unable to join monitor thread"); + } + } + log_debug("xmap", "monitor finished"); + } + + // finished + if (xconf.metadata_filename) { + json_metadata(xconf.metadata_file); + } + + if (xconf.output_module && xconf.output_module->close) { + xconf.output_module->close(&xconf, &xsend, &xrecv); + } + + if (xconf.probe_module && xconf.probe_module->close) { + xconf.probe_module->close(&xconf, &xsend, &xrecv); + } + +#ifdef PFRING + pfring_zc_destroy_cluster(xconf.pf.cluster); +#endif + + // blocklist free + blocklist_free(); + + // iterator free + iterator_free(it); + + log_info("xmap", "completed"); +} + +static void init_state() { + // init for xconf state + mpz_init(xconf.generator); + mpz_init(xconf.total_allowed_ip_port); + mpz_init(xconf.total_disallowed_ip_port); + mpz_init(xconf.total_allowed_ip); + mpz_init(xconf.total_disallowed_ip); + mpz_init(xconf.total_allowed_ip_port_actual); + + // init for xsend state + mpz_init(xsend.first_scanned); + mpz_init(xsend.max_index); +} + +static void deinit_state() { + // deinit for xconf state + mpz_clear(xconf.generator); + mpz_clear(xconf.total_allowed_ip_port); + mpz_clear(xconf.total_disallowed_ip_port); + mpz_clear(xconf.total_allowed_ip); + mpz_clear(xconf.total_disallowed_ip); + mpz_clear(xconf.total_allowed_ip_port_actual); + + // deinit for xsend state + mpz_clear(xsend.first_scanned); + mpz_clear(xsend.max_index); +} + +#define SET_IF_GIVEN(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = args.ARG##_arg; \ + }; \ + } +#define SET_BOOL(DST, ARG) \ + { \ + if (args.ARG##_given) { \ + (DST) = 1; \ + }; \ + } + +int main(int argc, char *argv[]) { + struct gengetopt_args_info args; + struct cmdline_parser_params *params; + params = cmdline_parser_params_create(); + params->initialize = 1; + params->override = 0; + params->check_required = 0; + + // parameters parser + int config_loaded = 0; + if (cmdline_parser_ext(argc, argv, &args, params) != 0) { + exit(EXIT_SUCCESS); + } + + if (args.config_given || file_exists(args.config_arg)) { + params->initialize = 0; + params->override = 0; + if (cmdline_parser_config_file(args.config_arg, &args, params) != 0) { + exit(EXIT_FAILURE); + } + config_loaded = 1; + } + + // initialize logging. if no log file or log directory are specified + // default to using stderr. + xconf.log_level = args.verbosity_arg; + xconf.log_file = args.log_file_arg; + xconf.log_directory = args.log_directory_arg; + + if (args.disable_syslog_given) { + xconf.syslog = 0; + } else { + xconf.syslog = 1; + } + + if (xconf.log_file && xconf.log_directory) { + log_init(stderr, xconf.log_level, xconf.syslog, "xmap"); + log_fatal("xmap", "log-file and log-directory cannot " + "specified simultaneously."); + } + + FILE *log_location = NULL; + if (xconf.log_file) { + log_location = fopen(xconf.log_file, "w"); + } else if (xconf.log_directory) { + time_t now; + time(&now); + struct tm *local = localtime(&now); + char path[100]; + strftime(path, 100, "xmap-%Y-%m-%dT%H%M%S%z.log", local); + char *fullpath = + xmalloc(strlen(xconf.log_directory) + strlen(path) + 2); + sprintf(fullpath, "%s/%s", xconf.log_directory, path); + log_location = fopen(fullpath, "w"); + free(fullpath); + } else { + log_location = stderr; + } + + if (!log_location) { + log_init(stderr, xconf.log_level, xconf.syslog, "xmap"); + log_fatal("xmap", "unable to open specified log file: %s", + strerror(errno)); + } + log_init(log_location, xconf.log_level, xconf.syslog, "xmap"); + + // xmap start + log_debug("xmap", "xmap main thread started"); + if (config_loaded) { + xconf.config_filename = args.config_arg; + log_debug("xmap", "loaded configuration file %s", args.config_arg); + } + + if (xconf.syslog) { + log_debug("xmap", "syslog support enabled"); + } else { + log_debug("xmap", "syslog support disabled"); + } + + // IPv46 parser + if (!args.ipv6_given && !args.ipv4_given) { + log_debug( + "xmap", + "no `-6'|`-4' flag given, default select to send ipv6 packet"); + xconf.ipv46_flag = 6; + xconf.ipv46_bytes = 16; + xconf.ipv46_bits = 128; + } else if (args.ipv6_given) { + log_debug("xmap", "`-6' flag given, send ipv6 packet"); + xconf.ipv46_flag = 6; + xconf.ipv46_bytes = 16; + xconf.ipv46_bits = 128; + } else if (args.ipv4_given) { + log_debug("xmap", "`-4' flag given, send ipv4 packet"); + xconf.ipv46_flag = 4; + xconf.ipv46_bytes = 4; + xconf.ipv46_bits = 32; + } + + // Scanning max bit length + if (args.max_len_given) { + enforce_range("scanning max-len", args.max_len_arg, 0, + xconf.ipv46_bits); + xconf.max_probe_len = args.max_len_arg; + if (xconf.max_probe_len <= 0 || + xconf.max_probe_len > xconf.ipv46_bits) { + log_fatal("xmap", "invalid max probe length: `-x|--max-len=%d'", + xconf.max_probe_len); + } + + xconf.max_probe_port_len = xconf.max_probe_len; + log_debug("xmap", "ipv%d max probing len=%d, +port=%d", + xconf.ipv46_flag, xconf.max_probe_len, + xconf.max_probe_port_len); + } else { + xconf.max_probe_len = 32; + xconf.max_probe_port_len = 32; + log_debug("xmap", + "no `-x|--max-len=' given, default select to scan all /%d " + "for ipv%d", + xconf.max_probe_len, xconf.ipv46_flag); + } + + // init state args + init_state(); + + // parse the provided probe and output module s.t. that we can support + // other command-line helpers (e.g. probe help) + // Output module + if (!strcmp(args.output_module_arg, "default")) { + log_debug("xmap", "no output module provided. will use `csv'."); + xconf.output_module = get_output_module_by_name("csv"); + xconf.output_module_name = strdup("csv"); + } else { + xconf.output_module = get_output_module_by_name(args.output_module_arg); + if (!xconf.output_module) { + log_fatal("xmap", "specified output module (%s) does not exist\n", + args.output_module_arg); + } + xconf.output_module_name = strdup(args.output_module_arg); + } + SET_IF_GIVEN(xconf.output_args, output_args); + + // Probe module + if (!strcmp(args.probe_module_arg, "default")) { + log_debug("xmap", "no probe module provided. will use `icmp_echo'."); + xconf.probe_module = + get_probe_module_by_name("icmp_echo", xconf.ipv46_flag); + } else { + xconf.probe_module = + get_probe_module_by_name(args.probe_module_arg, xconf.ipv46_flag); + if (!xconf.probe_module) { + log_fatal("xmap", + "specified IPv%d probe module (%s) does not exist\n", + xconf.ipv46_flag, args.probe_module_arg); + } + } + SET_IF_GIVEN(xconf.probe_args, probe_args); + SET_IF_GIVEN(xconf.probe_ttl, probe_ttl); + // check whether the probe module is going to generate dynamic data + // and that the output module can support exporting that data out of + // xmap. If they can't, then quit. + if (xconf.probe_module->output_type == OUTPUT_TYPE_DYNAMIC && + !xconf.output_module->supports_dynamic_output) { + log_fatal("xmap", + "specified probe module (%s) requires dynamic " + "output support, which output module (%s) does not support. " + "Most likely you want to use JSON output.", + args.probe_module_arg, args.output_module_arg); + } + + // iid module + if (!strcmp(args.iid_module_arg, "default")) { + log_debug("xmap", "no iid module provided. will use `low'."); + xconf.iid_module = get_iid_module_by_name("low"); + xconf.iid_module_name = strdup("low"); + } else { + xconf.iid_module = get_iid_module_by_name(args.iid_module_arg); + if (!xconf.iid_module) { + log_fatal("xmap", "specified iid module (%s) does not exist\n", + args.iid_module_arg); + } + xconf.iid_module_name = strdup(args.iid_module_arg); + } + if (args.iid_num_given && args.iid_num_arg < 0) { + log_fatal("xmap", "`--iid-num=' should > 0\n", args.iid_module_arg); + } + SET_IF_GIVEN(xconf.iid_args, iid_args); + SET_IF_GIVEN(xconf.iid_num, iid_num); + + // help output + if (args.help_given) { + cmdline_parser_print_help(); + + printf("\nProbe-module (IPv%d %s) Help:\n", xconf.ipv46_flag, + xconf.probe_module->name); + if (xconf.probe_module->helptext) { + fprintw(stdout, (char *) xconf.probe_module->helptext, 80); + } else { + printf("no help text available\n"); + } + + printf("\nOutput-module (%s) Help:\n", xconf.output_module->name); + if (!strcmp(args.output_module_arg, "default")) { + fprintw(stdout, (char *) default_help_text, 80); + } else if (xconf.output_module->helptext) { + fprintw(stdout, (char *) xconf.output_module->helptext, 80); + } else { + printf("no help text available\n"); + } + + printf("\nIID-module (%s) Help:\n", xconf.iid_module->name); + if (xconf.iid_module->helptext) { + fprintw(stdout, (char *) xconf.iid_module->helptext, 80); + } else { + printf("no help text available\n"); + } + printf("\n"); + exit(EXIT_SUCCESS); + } + + // other help + if (args.version_given) { + cmdline_parser_print_version(); + exit(EXIT_SUCCESS); + } + + if (args.list_output_modules_given) { + printf("Output-modules:\n"); + print_output_modules(); + exit(EXIT_SUCCESS); + } + + if (args.list_probe_modules_given) { + printf("Probe-modules (IPv6):\n"); + print_probe_modules(6); + printf("Probe-modules (IPv4):\n"); + print_probe_modules(4); + exit(EXIT_SUCCESS); + } + + if (args.list_iid_modules_given) { + printf("IID-modules:\n"); + print_iid_modules(); + exit(EXIT_SUCCESS); + } + + if (args.iplayer_given) { + xconf.send_ip_pkts = 1; + xconf.gw_mac_set = 1; + memset(xconf.gw_mac, 0, MAC_ADDR_LEN); + } + + if (cmdline_parser_required(&args, CMDLINE_PARSER_PACKAGE) != 0) { + exit(EXIT_FAILURE); + } + + //// fielddef set + // now that we know the probe module, let's find what it supports + memset(&xconf.fsconf, 0, sizeof(struct fieldset_conf)); + // the set of fields made available to a user is constructed + // of IP header fields + probe module fields + system fields + fielddefset_t *fds = &(xconf.fsconf.defs); + if (xconf.ipv46_flag == IPV4_FLAG) + gen_fielddef_set(fds, (fielddef_t *) &(ip_fields), ip_fields_len); + else if (xconf.ipv46_flag == IPV6_FLAG) + gen_fielddef_set(fds, (fielddef_t *) &(ip6_fields), ip6_fields_len); + gen_fielddef_set(fds, xconf.probe_module->fields, + xconf.probe_module->numfields); + gen_fielddef_set(fds, (fielddef_t *) &(sys_fields), sys_fields_len); + if (args.list_output_fields_given) { + printf("IPv%d %s:\n", xconf.ipv46_flag, xconf.probe_module->name); + for (int i = 0; i < fds->len; i++) { + printf("%-15s %6s: %s\n", fds->fielddefs[i].name, + fds->fielddefs[i].type, fds->fielddefs[i].desc); + } + exit(EXIT_SUCCESS); + } + + // find the fields we need for the framework + xconf.fsconf.success_index = fds_get_index_by_name(fds, (char *) "success"); + if (xconf.fsconf.success_index < 0) { + log_fatal("xmap", "probe module does not supply " + "required success field."); + } + xconf.fsconf.app_success_index = + fds_get_index_by_name(fds, (char *) "app_success"); + if (xconf.fsconf.app_success_index < 0) { + log_debug("xmap", "probe module does not supply " + "application success field."); + } else { + log_debug("xmap", + "probe module supplies app_success" + " output field. It will be included in monitor output"); + } + xconf.fsconf.classification_index = + fds_get_index_by_name(fds, (char *) "clas"); + if (xconf.fsconf.classification_index < 0) { + log_fatal("xmap", "probe module does not supply " + "required packet classification field."); + } + + // output field + // default output module does not support multiple fields throw an error + // if the user asks for this because otherwise we'll generate a + // malformed CSV file when this gets redirected to the CSV output module + if (args.output_fields_given && + !strcmp(args.output_module_arg, "default")) { + log_fatal("xmap", + "default output module does not support multiple" + " fields. Please specify an output module (e.g., -O csv)"); + } + // process the list of requested output fields. + if (args.output_fields_given) { + xconf.raw_output_fields = args.output_fields_arg; + } else if (!xconf.raw_output_fields) { + xconf.raw_output_fields = + (char *) "saddr"; // default output reply IPv6 address + } + // add all fields if wildcard received + if (!strcmp(xconf.raw_output_fields, "*")) { + xconf.output_fields_len = xconf.fsconf.defs.len; + xconf.output_fields = xcalloc(xconf.fsconf.defs.len, sizeof(char *)); + for (int i = 0; i < xconf.fsconf.defs.len; i++) { + xconf.output_fields[i] = + (char *) xconf.fsconf.defs.fielddefs[i].name; + } + fs_generate_full_fieldset_translation(&xconf.fsconf.translation, + &xconf.fsconf.defs); + } else { + split_string(xconf.raw_output_fields, &(xconf.output_fields_len), + &(xconf.output_fields)); + for (int i = 0; i < xconf.output_fields_len; i++) { + log_debug("xmap", "requested output field (%i): %s", i, + xconf.output_fields[i]); + } + // generate a translation that can be used to convert output + // from a probe module to the input for an output module + fs_generate_fieldset_translation( + &xconf.fsconf.translation, &xconf.fsconf.defs, xconf.output_fields, + xconf.output_fields_len); + } + + // output filter + // default filtering behavior is to drop unsuccessful and duplicates + if (!args.output_filter_arg || !strcmp(args.output_filter_arg, "default")) { + xconf.filter_duplicates = 1; + xconf.filter_unsuccessful = 1; + log_debug("xmap", + "no output filter specified. will use default: exclude " + "duplicates and unssuccessful"); + } else if (args.output_filter_arg && !strcmp(args.output_filter_arg, "")) { + xconf.filter_duplicates = 0; + xconf.filter_unsuccessful = 0; + log_debug("xmap", "empty output filter. will not exclude any values"); + } else { + // Run it through yyparse to build the expression tree + if (!parse_filter_string(args.output_filter_arg)) { + log_fatal("xmap", "unable to parse filter expression"); + } + // Check the fields used against the fieldset in use + if (!validate_filter(xconf.filter.expression, &xconf.fsconf.defs)) { + log_fatal("xmap", "invalid filter"); + } + xconf.output_filter_str = args.output_filter_arg; + log_debug("xmap", "will use output filter: %s", args.output_filter_arg); + } + + // find if xmap wants any specific cidrs scanned instead + // of the entire Internet + xconf.destination_cidrs = args.inputs; + xconf.destination_cidrs_len = (int) args.inputs_num; + SET_IF_GIVEN(xconf.blocklist_filename, blacklist_file); + SET_IF_GIVEN(xconf.allowlist_filename, whitelist_file); + if ((xconf.ipv46_flag == IPV4_FLAG) && xconf.blocklist_filename && + !strcmp(xconf.blocklist_filename, "/etc/xmap/blacklist4.conf")) { + log_warn( + "blocklist", + "XMap is currently using the default blacklist located at " + "/etc/xmap/blacklist4.conf. By default, this blacklist excludes " + "locally scoped networks (e.g. 10.0.0.0/8, 127.0.0.1/8, and " + "192.168.0.0/16). If you are trying to scan local networks, you " + "can change the default blacklist by editing the default XMap " + "configuration at /etc/xmap/xmap.conf."); + } + if ((xconf.ipv46_flag == IPV6_FLAG) && xconf.allowlist_filename && + !strcmp(xconf.allowlist_filename, "/etc/xmap/whitelist6.conf")) { + log_warn("allowlist", + "XMap is currently using the default whitelist located at " + "/etc/xmap/whitelist6.conf. By default, this whitelist " + "includes 2001::/3. If you are trying to scan other networks, " + "you can change the default whitelist by editing the default " + "XMap configuration at /etc/xmap/xmap.conf."); + } + + // target-port + if (xconf.probe_module->port_args) { + if (args.source_port_given) { + char *dash = strchr(args.source_port_arg, '-'); + if (dash) { // range + *dash = '\0'; + xconf.source_port_first = atoi(args.source_port_arg); + enforce_range("starting source-port", xconf.source_port_first, + 0, 0xFFFF); + xconf.source_port_last = atoi(dash + 1); + enforce_range("ending source-port", xconf.source_port_last, 0, + 0xFFFF); + if (xconf.source_port_first > xconf.source_port_last) { + fprintf(stderr, + "%s: invalid source port range: " + "last port is less than first port\n", + CMDLINE_PARSER_PACKAGE); + exit(EXIT_FAILURE); + } + } else { // single port + int port = atoi(args.source_port_arg); + enforce_range("source-port", port, 0, 0xFFFF); + xconf.source_port_first = port; + xconf.source_port_last = port; + } + } + log_debug("xmap", "src_port range: [%d, %d]", xconf.source_port_first, + xconf.source_port_last); + + if (!args.target_port_given) { + log_fatal("xmap", + "target port (-p) is required for this type of probe: %s", + xconf.probe_module->name); + } + parse_target_ports(args.target_port_arg); + init_target_port(); + } + + // iface + SET_IF_GIVEN(xconf.iface, interface); + + // mac + if (args.gateway_mac_given) { + if (!parse_mac(xconf.gw_mac, args.gateway_mac_arg)) { + fprintf(stderr, "%s: invalid MAC address `%s'\n", + CMDLINE_PARSER_PACKAGE, args.gateway_mac_arg); + exit(EXIT_FAILURE); + } + xconf.gw_mac_set = 1; + } + if (args.source_mac_given) { + if (!parse_mac(xconf.hw_mac, args.source_mac_arg)) { + fprintf(stderr, "%s: invalid MAC address `%s'\n", + CMDLINE_PARSER_PACKAGE, args.gateway_mac_arg); + exit(EXIT_FAILURE); + } + log_debug("send", + "source MAC address specified on CLI: " + "%02x:%02x:%02x:%02x:%02x:%02x", + xconf.hw_mac[0], xconf.hw_mac[1], xconf.hw_mac[2], + xconf.hw_mac[3], xconf.hw_mac[4], xconf.hw_mac[5]); + + xconf.hw_mac_set = 1; + } + + // source IP + if (args.source_ip_given) { + parse_source_ip_addresses(args.source_ip_arg); + } + + // rate + SET_IF_GIVEN(xconf.rate, rate); + + // bandwidth + if (args.bandwidth_given) { + // Supported: G,g=*1073741824; M,m=*1048576 K,k=*1024 bits per second + xconf.bandwidth = atoi(args.bandwidth_arg); + char *suffix = args.bandwidth_arg; + while (*suffix >= '0' && *suffix <= '9') { + suffix++; + } + if (*suffix) { + switch (*suffix) { + case 'G': + case 'g': + xconf.bandwidth *= 1073741824; // 1024^3 + break; + case 'M': + case 'm': + xconf.bandwidth *= 1048576; // 1024^2 + break; + case 'K': + case 'k': + xconf.bandwidth *= 1024; // 1024 + break; + default: + fprintf(stderr, + "%s: unknown bandwidth suffix '%s' (supported suffixes " + "are G, M and K)\n", + CMDLINE_PARSER_PACKAGE, suffix); + exit(EXIT_FAILURE); + } + } + } + + // batch + if (args.batch_given) { + xconf.batch = args.batch_arg; + if (xconf.batch < 0) { + log_fatal("xmap", "invalid batch number: `--batch='"); + } + } + + // packet_stream + SET_IF_GIVEN(xconf.packet_streams, probes); + if (xconf.packet_streams < 0) { + log_fatal("xmap", "invalid probe times: `--probes='"); + } + + // num-retries + xconf.num_retries = args.retries_arg; + if (xconf.num_retries < 0) { + log_fatal("xmap", "invalid retry count: `--retries='"); + } + + // cooldown seconds + SET_IF_GIVEN(xconf.cooldown_secs, cooldown_secs); + if (xconf.cooldown_secs < 0) { + log_fatal("xmap", "invalid cooldown secs: `-c|--cooldown-secs='"); + } + + // max target + SET_IF_GIVEN(xconf.max_targets, max_targets); + if (xconf.max_targets < 0) { + log_fatal("xmap", "invalid max target number: `-n|--max-targets='"); + } + if (xconf.max_targets) { + xsend.max_targets = xconf.max_targets; + } + + // max packet + SET_IF_GIVEN(xconf.max_packets, max_packets); + if (xconf.max_packets < 0) { + log_fatal("xmap", "invalid max packets number: `-k|--max-packets='"); + } + if (xconf.max_packets) { + xsend.max_packets = xconf.max_packets; + } + + // max runtime + SET_IF_GIVEN(xconf.max_runtime, max_runtime); + if (xconf.max_runtime < 0) { + log_fatal("xmap", "invalid max runtime number: `-t|--max-runtime='"); + } + + // max reuslts + SET_IF_GIVEN(xconf.max_results, max_results); + if (xconf.max_results < 0) { + log_fatal("xmap", "invalid max results number: `-N|--max-results='"); + } + + // est target + SET_IF_GIVEN(xconf.est_elements, est_elements); + if (xconf.est_elements < 0) { + log_fatal("xmap", "invalid est element number: `-E|--est-elements='"); + } + + // if there's a list of ips to scan, then initialize file reader + // blocklist_init should default allow all address space + // filename + SET_IF_GIVEN(xconf.output_filename, output_file); + SET_IF_GIVEN(xconf.list_of_ips_filename, list_of_ips_file); + if (xconf.list_of_ips_filename) { + log_debug("xmap", "init ip target file ing..."); + + int64_t count = ip_target_file_init(xconf.list_of_ips_filename); + if (count == -1) log_fatal("xmap", "init ip target file error"); + if (count == 0) { + log_warn("xmap", "zero ip target file, no need to xmap"); + return EXIT_SUCCESS; + } + xconf.list_of_ip_count = count; + + if (xconf.target_port_num) + xconf.list_of_ip_port_count = count * xconf.target_port_num; + else + xconf.list_of_ip_port_count = count; + + log_debug("xmap", "target ipv%d number: %d, number: %d", + xconf.ipv46_flag, count, xconf.ipv46_flag, + xconf.list_of_ip_port_count); + log_debug("xmap", "init ip target file completed"); + + xconf.max_probe_len = xconf.ipv46_bits; + xconf.max_probe_port_len = xconf.max_probe_len + xconf.target_port_bits; + log_debug("xmap", "load ip target from file, max probe len reset to %d", + xconf.max_probe_len); + } + xconf.ignore_filelist_error = args.ignore_filelist_error_given; + + // blocklist & allowlist + if ((xconf.ipv46_flag == IPV6_FLAG) && xconf.blocklist_filename && + !strcmp(xconf.blocklist_filename, "/etc/xmap/blacklist4.conf")) { + xconf.blocklist_filename = NULL; + } + if ((xconf.ipv46_flag == IPV4_FLAG) && xconf.allowlist_filename && + !strcmp(xconf.allowlist_filename, "/etc/xmap/whitelist6.conf")) { + xconf.allowlist_filename = NULL; + } + if (blocklist_init(xconf.allowlist_filename, xconf.blocklist_filename, + xconf.destination_cidrs, xconf.destination_cidrs_len, + NULL, 0, xconf.ignore_blacklist_error, + xconf.max_probe_len, xconf.target_port_bits, + xconf.ipv46_flag)) { + log_fatal("xmap", "unable to initialize blacklist/whitelist"); + } + xconf.ignore_blacklist_error = args.ignore_blacklist_error_given; + + // compute number of targets + blocklist_count_allowed_ip(xconf.total_allowed_ip); + if (mpz_zero(xconf.total_allowed_ip)) + log_fatal("xmap", "zero eligible addresses to scan"); + if (xconf.target_port_num) + mpz_mul_ui(xconf.total_allowed_ip_port_actual, xconf.total_allowed_ip, + xconf.target_port_num); + else + mpz_set(xconf.total_allowed_ip_port_actual, xconf.total_allowed_ip); + blocklist_count_not_allowed_ip(xconf.total_disallowed_ip); + blocklist_count_allowed_ip_port(xconf.total_allowed_ip_port); + blocklist_count_not_allowed_ip_port(xconf.total_disallowed_ip_port); + + // sender thread number +#ifndef PFRING + // Set the correct number of threads, default to num_cores - 1 + if (args.sender_threads_given) { + xconf.senders = args.sender_threads_arg; + } else { + xconf.senders = 1; + } + + if (((xsend.max_targets && 2 * xconf.senders >= xsend.max_targets) || + (xsend.max_packets && 2 * xconf.senders >= xsend.max_packets) || + (xconf.list_of_ips_filename && + 2 * xconf.senders >= xconf.list_of_ip_count) || + mpz_le_ui(xconf.total_allowed_ip_port, 2 * xconf.senders)) && + xconf.senders > 1) { + log_warn("xmap", + "too few targets relative to senders, dropping to one sender"); + xconf.senders = 1; + } +#else + xconf.senders = args.sender_threads_arg; +#endif + + // Figure out what cores to bind to + if (args.cores_given) { + char **core_list = NULL; + int len = 0; + split_string(args.cores_arg, &len, &core_list); + xconf.pin_cores_len = (uint32_t) len; + xconf.pin_cores = xcalloc(xconf.pin_cores_len, sizeof(uint32_t)); + for (uint32_t i = 0; i < xconf.pin_cores_len; ++i) + xconf.pin_cores[i] = atoi(core_list[i]); + } else { + int num_cores = sysconf(_SC_NPROCESSORS_ONLN); + xconf.pin_cores_len = (uint32_t) num_cores; + xconf.pin_cores = xcalloc(xconf.pin_cores_len, sizeof(uint32_t)); + for (uint32_t i = 0; i < xconf.pin_cores_len; ++i) + xconf.pin_cores[i] = i; + } + + // Check for a random seed + if (args.seed_given) { + xconf.seed = args.seed_arg; + xconf.seed_provided = 1; + } else { + // generate a seed randomly + if (!random_bytes(&xconf.seed, sizeof(uint64_t))) { + log_fatal("xmap", + "unable to generate random bytes needed for seed"); + } + xconf.seed_provided = 0; + } + xconf.aes = aesrand_init_from_seed(xconf.seed); + + // Set up sharding + xconf.shard_num = 0; + xconf.total_shards = 1; + if ((args.shard_given || args.shards_given) && !args.seed_given) + log_fatal("xmap", "need to specify seed if sharding a scan"); + if (args.shard_given ^ args.shards_given) + log_fatal( + "xmap", + "need to specify both shard number and total number of shards"); + if (args.shard_given) enforce_range("shard", args.shard_arg, 0, 65534); + if (args.shards_given) enforce_range("shards", args.shards_arg, 1, 65535); + SET_IF_GIVEN(xconf.shard_num, shard); + SET_IF_GIVEN(xconf.total_shards, shards); + if (xconf.shard_num >= xconf.total_shards) + log_fatal("xmap", + "with %hhu total shards, shard number (%hhu)" + " must be in range [0, %hhu)", + xconf.total_shards, xconf.shard_num, xconf.total_shards); + + // metadata + if (args.metadata_file_arg) { + xconf.metadata_filename = args.metadata_file_arg; + if (!strcmp(xconf.metadata_filename, "-")) { + xconf.metadata_file = stdout; + } else { + xconf.metadata_file = fopen(xconf.metadata_filename, "w"); + } + + if (!xconf.metadata_file) { + log_fatal("metadata", "unable to open metadata file (%s): %s", + xconf.metadata_filename, strerror(errno)); + } + log_debug("metadata", "metdata will be saved to %s", + xconf.metadata_filename); + } + if (args.user_metadata_given) { + xconf.custom_metadata_str = args.user_metadata_arg; + if (!json_tokener_parse(xconf.custom_metadata_str)) { + log_fatal("metadata", "unable to parse custom user metadata"); + } else { + log_debug("metadata", "user metadata validated successfully"); + } + } + if (args.notes_given) xconf.notes = args.notes_arg; + + // other + SET_IF_GIVEN(xconf.status_updates_file, status_updates_file); + SET_BOOL(xconf.dryrun, dryrun); + SET_BOOL(xconf.quiet, quiet); + SET_IF_GIVEN(xconf.max_sendto_failures, max_sendto_failures); + SET_IF_GIVEN(xconf.min_hitrate, min_hitrate); + + if (xconf.max_sendto_failures >= 0) + log_debug("xmap", + "scan will abort if more than %i sendto failures occur", + xconf.max_sendto_failures); + + if (xconf.min_hitrate > 0.0) + log_debug("xmap", "scan will abort if hitrate falls below %f", + xconf.min_hitrate); + + // start + start_xmap(); + + cmdline_parser_free(&args); + free(params); + deinit_state(); + + return EXIT_SUCCESS; +} diff --git a/src/xopt.ggo.in b/src/xopt.ggo.in new file mode 100644 index 0000000..0903080 --- /dev/null +++ b/src/xopt.ggo.in @@ -0,0 +1,223 @@ +# xmap option description to be processed by gengetopt + +package "xmap" +version "@XMAP_VERSION@" +purpose "A fast Internet-wide scanner for both IPv6 & IPv4." + +section "Basic arguments" +option "ipv6" 6 "Scanning the IPv6 networks (default)" + optional +option "ipv4" 4 "Scanning the IPv4 networks" + optional +option "max-len" x "Max IP bit length to scan" + typestr="len" + default="32" + optional int +option "target-port" p "Port(s) number to scan (for TCP and UDP scans), use `,' and `-', with this option, one target is a " + typestr="port|range" + optional string +option "output-file" o "Output file, use `-' for stdout" + typestr="name" + optional string +option "blacklist-file" b "File of subnets to exclude, in CIDR notation, e.g., 2001::/64, 192.168.0.0/16, www.qq.com/32 (max len of domain: 256)" + typestr="path" + optional string +option "whitelist-file" w "File of subnets to include, in CIDR notation, e.g., 2001::/64, 192.168.0.0/16, www.qq.com/32 (max len of domain: 256)" + typestr="path" + optional string +option "list-of-ips-file" I "List of individual addresses to scan in random order, e.g., 2001:db8::1, 192.168.0.1" + typestr="path" + optional string + +section "Scan options" +option "rate" R "Set send rate in pkts/sec" + typestr="pps" + optional int +option "bandwidth" B "Set send rate in bits/sec (supports suffixes G/g, M/m, and K/k)" + typestr="bps" + optional string +option "batch" - "Number of continuous targets probed before sleeping" + typestr="num" + default="1" + optional int +option "probes" - "Number of probes to send to each target" + typestr="num" + default="1" + optional int +option "retries" - "Max number of times to try to send packets if send fails" + typestr="num" + default="1" + optional int +option "max-targets" n "Capture number of targets to probe (default: no limit)" + typestr="num" + optional longlong +option "max-packets" k "Capture number of packets to send (default: no limit)" + typestr="num" + optional longlong +option "max-runtime" t "Capture length of time for sending packets (default: no limit)" + typestr="secs" + optional int +option "max-results" N "Capture number of results to return (default: no limit)" + typestr="num" + optional longlong +option "est-elements" E "Estimated number of results for unique, adapt to your memory capacity" + typestr="num" + default="500000000" + optional longlong +option "cooldown-secs" c "How long to continue receiving after sending last probe" + typestr="secs" + default="5" + optional int +option "seed" e "Seed used to select address permutation" + typestr="num" + optional longlong +option "shards" - "Set the total number of shards" + typestr="num" + optional int + default="1" +option "shard" - "Set which shard this scan is (0 indexed)" + typestr="num" + optional int + default="0" + +section "Network options" +option "source-port" s "Source port(s) for scan packets, use `-'" + typestr="port|range" + optional string +option "source-ip" S "Source address(es) for scan packets, use `,' and `-'" + typestr="ip|range" + optional string +option "gateway-mac" G "Specify gateway MAC address, e.g., 12:34:56:78:90:ab" + typestr="mac" + optional string +option "source-mac" - "Specify source MAC address, e.g., 12:34:56:78:90:ab" + typestr="mac" + optional string +option "interface" i "Specify network interface to use" + typestr="name" + optional string +option "iplayer" X "Sends IP packets instead of Ethernet (for VPNs)" + optional + +section "Probe Modules" +option "probe-module" M "Select probe module (default=`icmp_echo')" + typestr="name" + default="default" + optional string +option "probe-args" - "Arguments to pass to probe module" + typestr="args" + optional string +option "probe-ttl" - "Set TTL value for probe IP packets" + typestr="hops" + default="255" + optional int +option "list-probe-modules" - "List available probe modules" + optional + +section "Output Modules" +option "output-module" O "Select output module (default=`csv')" + typestr="name" + default="default" + optional string +option "output-args" - "Arguments to pass to output module" + typestr="args" + optional string +option "output-fields" f "Fields that should be output in result set, use `,' and `*'" + typestr="fields" + optional string +option "output-filter" - "Specify a filter over the response fields to limit what responses get sent to the output module" + typestr="filter" + optional string +option "list-output-modules" - "List available output modules" + optional +option "list-output-fields" - "List all fields that can be output by selected probe module" + optional + +section "IID Modules" +option "iid-module" U "Select iid module (default=`low')" + typestr="name" + default="default" + optional string +option "iid-args" - "Arguments to pass to iid module" + typestr="args" + optional string +option "iid-num" - "Number of iid for one IP prefix" + typestr="num" + default="1" + optional int +option "list-iid-modules" - "List available iid modules" + optional + +section "Logging and Metadata" +option "quiet" q "Do not print status updates" + optional +option "verbosity" v "Level of log detail (0-5)" + typestr="num" + default="3" + optional int +option "log-file" l "Write log entries to file" + typestr="name" + optional string +option "log-directory" L "Write log entries to a timestamped file in this directory" + typestr="path" + optional string +option "metadata-file" m "Output file for scan metadata (JSON)" + typestr="name" + optional string +option "status-updates-file" u "Write scan progress updates to CSV file" + typestr="name" + optional string +option "disable-syslog" - "Disables logging messages to syslog" + optional +option "notes" - "Inject user-specified notes into scan metadata" + typestr="notes" + optional string +option "user-metadata" - "Inject user-specified JSON metadata into scan metadata" + typestr="json" + optional string + +section "Additional options" +option "sender-threads" T "Threads used to send packets" + typestr="num" + default="1" + optional int +option "config" C "Read a configuration file, which can specify any of these options" + typestr="filename" + default="/etc/xmap/xmap.conf" + optional string +option "dryrun" d "Do not actually send packets, just show" + optional +option "max-sendto-failures" - "Maximum NIC sendto failures before scan is aborted" + typestr="num" + default="-1" + optional int +option "min-hitrate" - "Minimum hitrate that scan can hit before scan is aborted" + typestr="rate" + default="0.0" + optional float +option "cores" - "Comma-separated list of cores to pin to" + optional string +option "ignore-blacklist-error" - "Ignore invalid entries in `--whitelist-file/blacklist-file'" + optional +option "ignore-filelist-error" - "Ignore invalid entries in `--list-of-ips-file'" + optional +option "help" h "Print help and exit" + optional +option "version" V "Print version and exit" + optional + +text "\nExamples: + xmap + scan the ::/0-32 space by Echo ping and output to stdout + xmap -4 + scan the 0.0.0.0/0-32 space by Echo ping and output to stdout + xmap -N 5 -B 10M + find 5 alive IPv6 hosts, scanning at 10 Mb/s + xmap 2001::/8 2002::/16 + scan both subnets for 2001::/8-32 and 2002::/16-32 space + xmap -x 64 2001::/32 -U rand + scan 2001::/32-64 with random IID, e.g., 2001::1783:ab42:9247:cb38 + xmap -M icmp_echo -O csv -U low -h + show help text for modules icmp_echo, csv, and low + xmap -M tcp_syn -p 80,443,8080-8081 + scan the ::/0-32 space for port 80,443,8080,8081 by TCP SYN ping" diff --git a/src/xopt_include.c b/src/xopt_include.c new file mode 100644 index 0000000..5023fa6 --- /dev/null +++ b/src/xopt_include.c @@ -0,0 +1,20 @@ +/* + * ZMap Copyright 2013 Regents of the University of Michigan + * + * XMap Copyright 2021 Xiang Li from Network and Information Security Lab + * Tsinghua University + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + */ + +#if __GNUC__ < 4 +#error "gcc version >= 4 is required" +#elif __GNUC__ == 4 && __GNUC_MINOR__ >= 6 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#elif __GNUC_MINOR__ >= 4 +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +#include "xopt.c"