diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bb04f6aa4..5cbcb8d68 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,88 +41,25 @@ jobs: poetry install - name: Run pytest run: | - poetry run make test + poetry run make pytest - vecu: + bats: strategy: fail-fast: false - matrix: - python-version: ['3.11', '3.12'] runs-on: ubuntu-latest + container: debian:stable steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - uses: Gr1N/setup-poetry@v9 - - uses: actions/cache@v4 + - uses: actions/cache@v3 with: path: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }} - - name: Install Dependencies run: | + apt-get update -y && apt-get install -y bats python3 python3-poetry jq zstd poetry install - - - name: Spawn vECU - run: | - poetry run gallia script vecu "unix-lines:///tmp/vecu.sock" rng --seed 3 --mandatory_sessions "[1, 2, 3]" --mandatory_services "[DiagnosticSessionControl, EcuReset, ReadDataByIdentifier, WriteDataByIdentifier, RoutineControl, SecurityAccess, ReadMemoryByAddress, WriteMemoryByAddress, RequestDownload, RequestUpload, TesterPresent, ReadDTCInformation, ClearDiagnosticInformation, InputOutputControlByIdentifier]" & - - - name: Add config - run: | - echo "[gallia]" > gallia.toml - echo "[gallia.scanner]" >> gallia.toml - echo 'target = "unix-lines:///tmp/vecu.sock"' >> gallia.toml - echo 'dumpcap = false' >> gallia.toml - echo "[gallia.protocols.uds]" >> gallia.toml - echo 'ecu_reset = 0x01' >> gallia.toml - - - name: Dump Config and Defaults - run: | - poetry run gallia --show-config - poetry run gallia --show-defaults - - - name: Test scan-services - run: | - poetry run gallia scan uds services --sessions 1 2 --check-session - - - name: Test scan-sessions - run: | - poetry run gallia scan uds sessions --depth 2 - poetry run gallia scan uds sessions --fast - - - name: Test scan-identifiers - run: | - poetry run gallia scan uds identifiers --start 0 --end 100 --sid 0x22 - poetry run gallia scan uds identifiers --start 0 --end 100 --sid 0x2e - poetry run gallia scan uds identifiers --start 0 --end 100 --sid 0x31 - - - name: Test scan-reset - run: | - poetry run gallia scan uds reset - - - name: Test scan-dump-seeds - run: | - poetry run gallia scan uds dump-seeds --duration 0.01 --level 0x2f - - - name: Test scan-memory-functions - run: | - for sid in 0x23 0x34 0x35 0x3d; do - poetry run gallia scan uds memory --sid "$sid" - done - - - name: Test UDS primitives + - name: Run bats run: | - poetry run gallia primitive uds ecu-reset - poetry run gallia primitive uds vin - poetry run gallia primitive uds ping --count 2 - poetry run gallia primitive uds rdbi 0x108d - poetry run gallia primitive uds pdu 1001 - poetry run gallia primitive uds wdbi 0x2266 --data 00 - poetry run gallia primitive uds dtc read - poetry run gallia primitive uds dtc clear - poetry run gallia primitive uds dtc control --stop - poetry run gallia primitive uds dtc control --resume - poetry run gallia primitive uds iocbi 0x1000 reset-to-default + poetry run make bats diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 000000000..839f507b1 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: CC0-1.0 + +shell=bash diff --git a/Makefile b/Makefile index ca4399527..c3b692f03 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,9 @@ default: @echo " fmt run autoformatters" @echo " lint run linters" @echo " docs build docs" - @echo " test run testsuite" + @echo " tests run testsuite" + @echo " pytest run pytest tests" + @echo " bats run bats end to end tests" @echo " clean delete build artifacts" .PHONY: zipapp @@ -26,6 +28,7 @@ lint: mypy src tests ruff check src tests ruff format --check src tests + find tests/bats \( -iname "*.bash" -or -iname "*.bats" -or -iname "*.sh" \) | xargs shellcheck reuse lint .PHONY: lint-win32 @@ -35,16 +38,24 @@ lint-win32: .PHONY: fmt fmt: - ruff check --fix-only src tests - ruff format src tests + ruff check --fix-only src tests/pytest + ruff format src tests/pytest + find tests/bats \( -iname "*.bash" -or -iname "*.bats" -or -iname "*.sh" \) | xargs shfmt -w .PHONY: docs docs: $(MAKE) -C docs html -.PHONY: test -test: - python -m pytest -v --cov=$(PWD) --cov-report html tests +.PHONY: tests +tests: pytest bats + +.PHONY: pytest +pytest: + python -m pytest -v --cov=$(PWD) --cov-report html tests/pytest + +.PHONY: bats +bats: + ./tests/bats/run_bats.sh .PHONY: clean clean: diff --git a/flake.nix b/flake.nix index f38530438..058d5be1a 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,10 @@ devShell.x86_64-linux = pkgs.mkShell { buildInputs = with pkgs; [ poetry + shellcheck + shfmt + bats + nodePackages_latest.bash-language-server python311 python312 ]; diff --git a/src/hr/__init__.py b/src/hr/__init__.py index f322f65ed..e8001d843 100644 --- a/src/hr/__init__.py +++ b/src/hr/__init__.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 import argparse +import os +import signal import sys from itertools import islice from pathlib import Path @@ -62,7 +64,7 @@ def _main() -> int: for file in args.FILE: path = cast(Path, file) - if not (path.is_file() or path.is_fifo() or str(path) != "-"): + if not (path.is_file() or path.is_fifo() or str(path) == "-"): print(f"not a regular file: {path}", file=sys.stderr) return 1 @@ -87,9 +89,18 @@ def main() -> None: sys.exit(_main()) except (msgspec.DecodeError, msgspec.ValidationError) as e: print(f"invalid file format: {e}", file=sys.stderr) + sys.exit(1) # BrokenPipeError appears when stuff is piped to | head. - except (KeyboardInterrupt, BrokenPipeError): - pass + # This is not an error for hr. + except BrokenPipeError: + # https://docs.python.org/3/library/signal.html#note-on-sigpipe + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown. + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(0) + except KeyboardInterrupt: + sys.exit(128 + signal.SIGINT) if __name__ == "__main__": diff --git a/tests/bats/001-invocation.bats b/tests/bats/001-invocation.bats new file mode 100644 index 000000000..99e8750b2 --- /dev/null +++ b/tests/bats/001-invocation.bats @@ -0,0 +1,22 @@ +#!/usr/bin/env bats + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +load helpers.bash + +@test "invoke gallia without parameters" { + # Should fail and print help page. + run -64 gallia +} + +@test "invoke gallia without config" { + run -1 gallia --show-config +} + +@test "invoke gallia with config" { + setup_gallia_toml + gallia --show-config + rm_gallia_toml +} diff --git a/tests/bats/002-scans.bats b/tests/bats/002-scans.bats new file mode 100644 index 000000000..9281bbc06 --- /dev/null +++ b/tests/bats/002-scans.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +load helpers.bash + +setup_file() { + setup_gallia_toml +} + +@test "scan services" { + gallia scan uds services --sessions 1 2 --check-session +} + +@test "scan sessions" { + gallia scan uds sessions --depth 2 +} + +@test "scan fast" { + gallia scan uds sessions --fast +} + +@test "scan identifiers sid 0x22" { + gallia scan uds identifiers --start 0 --end 100 --sid 0x22 +} + +@test "scan identifiers sid 0x2e" { + gallia scan uds identifiers --start 0 --end 100 --sid 0x2e +} + +@test "scan identifiers sid 0x31" { + gallia scan uds identifiers --start 0 --end 100 --sid 0x31 +} + +@test "scan reset" { + gallia scan uds reset +} + +@test "scan dump-seeds" { + gallia scan uds dump-seeds --duration 0.01 --level 0x2f +} + +@test "scan memory" { + for sid in 0x23 0x34 0x35 0x3d; do + gallia scan uds memory --sid "$sid" + done +} diff --git a/tests/bats/003-primitives.bats b/tests/bats/003-primitives.bats new file mode 100644 index 000000000..bdadf782e --- /dev/null +++ b/tests/bats/003-primitives.bats @@ -0,0 +1,55 @@ +#!/usr/bin/env bats + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +load helpers.bash + +setup_file() { + setup_gallia_toml +} + +@test "primitive ecu-reset" { + gallia primitive uds ecu-reset +} + +@test "primitive vin" { + gallia primitive uds vin +} + +@test "primitive ping" { + gallia primitive uds ping --count 2 +} + +@test "primitive rdbi" { + gallia primitive uds rdbi 0x108d +} + +@test "primitive pdu" { + gallia primitive uds pdu 1001 +} + +@test "primitive wdbi" { + gallia primitive uds wdbi 0x2266 --data 00 +} + +@test "primitive dtc read" { + gallia primitive uds dtc read +} + +@test "primitive dtc clear" { + gallia primitive uds dtc clear +} + +@test "primitive dtc control stop" { + gallia primitive uds dtc control --stop +} + +@test "primitive dtc control resume" { + gallia primitive uds dtc control --resume +} + +@test "primitive iocbi reset-to-default" { + gallia primitive uds iocbi 0x1000 reset-to-default +} diff --git a/tests/bats/100-hr.bats b/tests/bats/100-hr.bats new file mode 100644 index 000000000..c2fa663e1 --- /dev/null +++ b/tests/bats/100-hr.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +@test "read logs from .zst file" { + hr "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" +} + +@test "read logs from .gz file" { + zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | gzip - >"$BATS_TMPDIR/log-01.json.gz" + hr "$BATS_TMPDIR/log-01.json.gz" +} + +@test "read logs from stdin" { + zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | hr - +} + +@test "read multiple .zst files" { + hr "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" +} + +@test "read file with priority prefix" { + zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | awk '{print "<6>" $0}' | hr - +} + +@test "pipe invalid data" { + run -1 bash -c "echo 'invalid json' | hr -" +} + +@test "pipe to head and handle SIGPIPE" { + hr "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | head +} + +@test "filter priority" { + local additional_line + additional_line='{"module": "foo", "data": "I am the line!", "host": "kronos", "datetime":"2020-04-23T15:21:50.620310", "priority": 5, "version": 2}' + cat <(zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst") <(echo "$additional_line") | gzip - >"$BATS_TMPDIR/log.json.gz" + + run -0 hr -p notice "$BATS_TMPDIR/log.json.gz" + + [[ "$output" =~ "I am the line!" ]] +} + +@test "filter priority with priority prefix" { + local additional_line + additional_line='<5>{"module": "foo", "data": "I am the line!", "host": "kronos", "datetime":"2020-04-23T15:21:50.620310", "priority": 5, "version": 2}' + cat <(zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst") <(echo "$additional_line") | gzip - >"$BATS_TMPDIR/log.json.gz" + + run -0 hr -p notice "$BATS_TMPDIR/log.json.gz" + + [[ "$output" =~ "I am the line!" ]] +} diff --git a/tests/bats/README.md b/tests/bats/README.md new file mode 100644 index 000000000..c93f30421 --- /dev/null +++ b/tests/bats/README.md @@ -0,0 +1,18 @@ + + +# Bats Testsuite + +This directory contains code which is part of the `gallia` testsuite using [`bats`](https://bats-core.readthedocs.io). + +* 0XX: Tests for the tool `gallia` +* 1XX: Tests for the tool `hr` + +## Run the Bats Testsuite + +A virtual ECU must be online and listening on the unix socket `/tmp/vecu.sock`. +`run_bats.sh` takes care of starting the virtual ECU. +If the testsuite is started plain, then the virtual ECU needs to be started separately. diff --git a/tests/bats/helpers.bash b/tests/bats/helpers.bash new file mode 100644 index 000000000..997e1aa9f --- /dev/null +++ b/tests/bats/helpers.bash @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +setup_gallia_toml() { + { + echo "[gallia]" + echo "no-volatile-info = true" + echo "verbosity = 1" + + echo "[gallia.scanner]" + echo 'target = "unix-lines:///tmp/vecu.sock"' + echo 'dumpcap = false' + + echo "[gallia.protocols.uds]" + echo 'ecu_reset = 0x01' + } >"$BATS_FILE_TMPDIR/gallia.toml" + + export GALLIA_CONFIG="$BATS_FILE_TMPDIR/gallia.toml" +} + +rm_gallia_toml() { + if [[ -r "$GALLIA_CONFIG" ]]; then + rm -f "$GALLIA_CONFIG" + fi +} diff --git a/tests/bats/run_bats.sh b/tests/bats/run_bats.sh new file mode 100755 index 000000000..31c8368d7 --- /dev/null +++ b/tests/bats/run_bats.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: CC0-1.0 + +set -eu + +gallia script vecu --no-volatile-info "unix-lines:///tmp/vecu.sock" rng \ + --seed 3 \ + --mandatory_sessions "[1, 2, 3]" \ + --mandatory_services "[DiagnosticSessionControl, EcuReset, ReadDataByIdentifier, WriteDataByIdentifier, RoutineControl, SecurityAccess, ReadMemoryByAddress, WriteMemoryByAddress, RequestDownload, RequestUpload, TesterPresent, ReadDTCInformation, ClearDiagnosticInformation, InputOutputControlByIdentifier]" 2>vecu.log & + +# https://superuser.com/a/553236 +trap 'kill "$(jobs -p)"' SIGINT SIGTERM EXIT + +if ! bats -r "$(dirname "$BASH_ARGV0")"; then + cat vecu.log + exit 1 +fi diff --git a/tests/bats/setup_suite.bash b/tests/bats/setup_suite.bash new file mode 100644 index 000000000..43a90b7fb --- /dev/null +++ b/tests/bats/setup_suite.bash @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +setup_suite() { + bats_require_minimum_version 1.8.0 + + # https://bats-core.readthedocs.io/en/stable/tutorial.html#let-s-do-some-setup + DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" >/dev/null 2>&1 && pwd)" + PATH="$DIR/..:$PATH" + + cd "$BATS_TEST_TMPDIR" || exit 1 +} diff --git a/tests/bats/testfiles/log-01.json.zst b/tests/bats/testfiles/log-01.json.zst new file mode 100644 index 000000000..61c68e191 Binary files /dev/null and b/tests/bats/testfiles/log-01.json.zst differ diff --git a/tests/bats/testfiles/log-01.json.zst.license b/tests/bats/testfiles/log-01.json.zst.license new file mode 100644 index 000000000..75b05f631 --- /dev/null +++ b/tests/bats/testfiles/log-01.json.zst.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: AISEC Pentesting Team + +SPDX-License-Identifier: CC0-1.0 diff --git a/tests/test_config.py b/tests/pytest/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/pytest/test_config.py diff --git a/tests/test_help.py b/tests/pytest/test_help.py similarity index 100% rename from tests/test_help.py rename to tests/pytest/test_help.py diff --git a/tests/test_helpers.py b/tests/pytest/test_helpers.py similarity index 100% rename from tests/test_helpers.py rename to tests/pytest/test_helpers.py diff --git a/tests/test_target_uris.py b/tests/pytest/test_target_uris.py similarity index 100% rename from tests/test_target_uris.py rename to tests/pytest/test_target_uris.py diff --git a/tests/test_transports.py b/tests/pytest/test_transports.py similarity index 100% rename from tests/test_transports.py rename to tests/pytest/test_transports.py