From 17d6a8cf0d9b9e37b7baecfc827a4a84c554e1c6 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 31 Dec 2024 19:41:07 +0000 Subject: [PATCH 1/4] Update project to follow template --- .cruft.json | 23 ++++++ .dockerignore | 14 +++- .gitignore | 144 +++++++++++++++++++++++++++++++++++-- .vscode/launch.json | 21 ++++++ .vscode/settings.json | 5 -- CONTRIBUTING.md | 56 +++++++++++++-- LICENCE.md | 11 --- LICENSE | 26 +++++++ Makefile | 26 +++++++ README.md | 92 +++++++++++++++++++++--- pipelines/.cruft.json | 2 +- pipelines/azure-tests.yaml | 3 +- pkglist.txt | 0 pyproject.toml | 2 + requirements.txt | 4 +- service_manifest.yml | 13 ++-- tests/gentests.py | 30 ++++++++ tests/gentests.sh | 24 +++++++ tests/pytest.sh | 24 +++++++ tests/requirements.txt | 1 + 20 files changed, 475 insertions(+), 46 deletions(-) create mode 100644 .cruft.json create mode 100644 .vscode/launch.json delete mode 100644 LICENCE.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 pkglist.txt create mode 100644 pyproject.toml create mode 100755 tests/gentests.py create mode 100755 tests/gentests.sh create mode 100755 tests/pytest.sh diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 0000000..7020d61 --- /dev/null +++ b/.cruft.json @@ -0,0 +1,23 @@ +{ + "template": "https://github.com/CybercentreCanada/assemblyline-service-template.git", + "commit": "6a32666117e30dbfb990bfaf6e7c9eb25badd03a", + "checkout": null, + "context": { + "cookiecutter": { + "service_name": "suricata", + "__svc_name": "suricata", + "__repository": "assemblyline-service-suricata", + "__pkg_name": "suricata", + "__class_name": "Suricata", + "short_description": "This service scans network capture files with signature and extract files from network capture.", + "short_description_fr": "Ce service analyse les fichiers de capture réseau avec signature et extrait les fichiers de la capture réseau.", + "stage": "CORE", + "category": "Networking", + "org_name_full": "CybercentreCanada", + "org_name_short": "cccs", + "license": "mit", + "_template": "https://github.com/CybercentreCanada/assemblyline-service-template.git" + } + }, + "directory": null +} diff --git a/.dockerignore b/.dockerignore index a764cee..5f78d7b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,25 @@ Dockerfile .idea .git +.gitignore +.vscode +.dockerignore pipelines venv +.venv env +.env test tests -exemples +examples docs +build +dist +**/__pycache__ +**/*.pyc + pip-log.txt pip-delete-this-directory.txt .tox @@ -18,5 +28,5 @@ pip-delete-this-directory.txt .cache nosetests.xml coverage.xml -*,cover +*.cover *.log diff --git a/.gitignore b/.gitignore index 9c07ba8..78f6696 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=python,vim,visualstudiocode + +### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -6,11 +10,6 @@ __pycache__/ # C extensions *.so -# IDE files -.pydevproject -.python-version -.idea - # Distribution / packaging .Python build/ @@ -60,6 +59,22 @@ cover/ *.mo *.pot +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + # PyBuilder .pybuilder/ target/ @@ -71,6 +86,43 @@ target/ profile_default/ ipython_config.py +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + # Environments .env .venv @@ -80,5 +132,85 @@ ENV/ env.bak/ venv.bak/ +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + # Cython debug symbols -cython_debug/ \ No newline at end of file +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..b4f3881 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "RunServiceOnce", + "type": "python", + "request": "launch", + "module": "assemblyline_v4_service.dev.run_service_once", + "cwd": "${workspaceFolder}", + "args": [ + "-d", + "suricata_.suricata_.Suricata", + "${file}" + ], + "justMyCode": false, + }, + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index aa60af9..54dc624 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -29,9 +29,4 @@ //Added the ignore of E203 for now : https://github.com/PyCQA/pycodestyle/issues/373 "--ignore=E203,W503" ], - "python.testing.pytestArgs": [ - "tests" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0a60cc..b9785be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,24 +2,68 @@ This guide covers the basics of how to contribute to the Assemblyline project. -Python code should follow the PEP8 guidelines defined here: [PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/). +Python code should follow the PEP8 guidelines defined here: +[PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/). ## Tell us want you want to build/fix -Before you start coding anything you should connect with the Assemblyline community via the [Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the [central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision of the system. + +Before you start coding anything you should connect with the Assemblyline community via the +[Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the +[central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one +else is working on the same thing and that whatever you are going to build still fits with the vision of the system. ## Git workflow - Clone the repo to your own account - Checkout and pull the latest commits from the master branch - Make a branch -- Work in any way you like and make sure your changes actually work -- When you're satisfied with your changes, create a pull requests to the main assemblyline repo +- Work on your modifications and make sure your changes work as expected +- When you're satisfied with your changes, create a pull requests to the Assemblyline repo #### Transfer your service repo -If you've worked on a new service that you want to be included in the default service selection you'll have to transfer the repo into our control. + +If you've worked on a new service that you want to be included in the default service selection you'll have to transfer +the associated repo into our control. #### You are not allow to merge: -Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge code into our repositories. +Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge +code into our repositories. We check for new pull requests every day and will merge them in once they have been approved by someone in our team. + +# Guide de contribution d'Assemblyline + +Ce guide couvre les bases de la façon de contribuer au projet Assemblyline. + +Le code Python doit suivre les directives PEP8 définies ici: +[Directives PEP8](https://www.python.org/dev/peps/pep-0008/). + +## Dites-nous que vous voulez construire / réparer + +Avant de commencer à coder quoi que ce soit, vous devriez vous connecter à la communauté Assemblyline via le +[Serveur Discord Assemblyline](https://discord.gg/GUAy9wErNu) et/ou le +[projet GitHub central Assemblyline](https://github.com/CybercentreCanada/assemblyline/issues) pour vous assurer que +personne d'autre ne travaille sur la même chose et que tout ce que vous allez construire correspond toujours à la vision +du système. + +## Flux de travail avec Git + +- Clonez le référentiel sur votre propre compte +- Changez de branche pour la branche principale et la synchroniser avec le serveur de référence +- Faire une nouvelle branche +- Travaillez sur ce que vous souhaitez et assurez-vous que vos modifications fonctionnent comme prévu +- Lorsque vous êtes satisfait de vos modifications, créez une demande de fusion sur le référentiel d'Assemblyline + +#### Transférer votre référentiel de service + +Si vous avez travaillé sur un nouveau service que vous souhaitez inclure dans la sélection de service par défaut, vous +devrez transférer le référentiel associé sous notre contrôle. + +#### Vous n'êtes pas autorisé à compléter une fusion: + +Même si vous tentez de compléter une demande de fusion, vous serez refusé. Seules quelques personnes de notre équipe +sont autorisées à fusionner dans nos référentiels. + +Nous vérifions les nouvelles demande de fusion tous les jours et les fusionnerons une fois qu'elles auront été approuvées +par quelqu'un de notre équipe. diff --git a/LICENCE.md b/LICENCE.md deleted file mode 100644 index fd4ad66..0000000 --- a/LICENCE.md +++ /dev/null @@ -1,11 +0,0 @@ -MIT License - -Copyright (c) 2020 Crown Copyright, Government of Canada (Canadian Centre for Cyber Security / Communications Security Establishment) - -Copyright title to all 3rd party software distributed with Assemblyline (AL) is held by the respective copyright holders as noted in those files. Users are asked to read the 3rd Party Licenses referenced with those assets. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..22d1fa7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +MIT License + +Copyright (c) 2020 Crown Copyright, Government of Canada +(Canadian Centre for Cyber Security / Communications Security Establishment) + +Copyright title to all 3rd party software distributed with Assemblyline (AL) +is held by the respective copyright holders as noted in those files. Users +are asked to read the 3rd Party Licenses referenced with those assets. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b900e4a --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +ifndef VERSION +$(error VERSION is undefined) +endif + +TAG?=latest +ORG?=cccs + +ifneq ($(ORG)x, x) +ORG:=$(ORG)/ +endif +ifneq ($(REGISTRY)x, x) +ORG:=$(REGISTRY)/ +endif + +.PHONY: default +default: build + +.PHONY: build +build: + docker build \ + --pull \ + --build-arg version=$(VERSION) \ + --build-arg branch=stable \ + -t $(REGISTRY)$(ORG)assemblyline-service-suricata:$(TAG)\ + -f ./Dockerfile \ + . diff --git a/README.md b/README.md index 1219ddb..b5dd17a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ +[![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu) +[![](https://img.shields.io/discord/908084610158714900)](https://discord.gg/GUAy9wErNu) +[![Static Badge](https://img.shields.io/badge/github-assemblyline-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline) +[![Static Badge](https://img.shields.io/badge/github-assemblyline\_service\_suricata-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline-service-suricata) +[![GitHub Issues or Pull Requests by label](https://img.shields.io/github/issues/CybercentreCanada/assemblyline/service-suricata)](https://github.com/CybercentreCanada/assemblyline/issues?q=is:issue+is:open+label:service-suricata) +[![License](https://img.shields.io/github/license/CybercentreCanada/assemblyline-service-suricata)](./LICENSE) # Suricata Service This service scans network capture files with signature and extract files from network capture. -**NOTE**: This service does not require you to buy any licence and is preinstalled and working after a default installation - -## Execution +## Service Details The Suricata configuration file is available in suricata\_.conf.suricata.yaml. @@ -16,10 +20,82 @@ The ruleset(s) configured by default for use with this service are: Organizations can add their own rulesets to this service. -## Test if working +## Image variants and tags + +Assemblyline services are built from the [Assemblyline service base image](https://hub.docker.com/r/cccs/assemblyline-v4-service-base), +which is based on Debian 11 with Python 3.11. + +Assemblyline services use the following tag definitions: + +| **Tag Type** | **Description** | **Example Tag** | +| :----------: | :----------------------------------------------------------------------------------------------- | :------------------------: | +| latest | The most recent build (can be unstable). | `latest` | +| build_type | The type of build used. `dev` is the latest unstable build. `stable` is the latest stable build. | `stable` or `dev` | +| series | Complete build details, including version and build type: `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | + +## Running this service + +This is an Assemblyline service. It is designed to run as part of the Assemblyline framework. + +If you would like to test this service locally, you can run the Docker image directly from the a shell: + + docker run \ + --name Suricata \ + --env SERVICE_API_HOST=http://`ip addr show docker0 | grep "inet " | awk '{print $2}' | cut -f1 -d"/"`:5003 \ + --network=host \ + cccs/assemblyline-service-suricata + +To add this service to your Assemblyline deployment, follow this +[guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/run_your_service/#add-the-container-to-your-deployment). + +## Documentation + +General Assemblyline documentation can be found at: https://cybercentrecanada.github.io/assemblyline4_docs/ + +# Service Suricata + +Ce service analyse les fichiers de capture réseau avec signature et extrait les fichiers de la capture réseau. + +## Détails du service + +Le fichier de configuration de Suricata est disponible dans suricata\_.conf.suricata.yaml. + +Le(s) jeu(x) de règles configuré(s) par défaut pour être utilisé(s) avec ce service sont : + +- [Emerging Threats Open](https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz) +- [Snortv3 Community](https://www.snort.org/downloads/community/snort3-community-rules.tar.gz) +- [URLhaus](https://urlhaus.abuse.ch/downloads/urlhaus_suricata.tar.gz) + +Les organisations peuvent ajouter leurs propres jeux de règles à ce service. + +## Variantes et étiquettes d'image + +Les services d'Assemblyline sont construits à partir de l'image de base [Assemblyline service](https://hub.docker.com/r/cccs/assemblyline-v4-service-base), +qui est basée sur Debian 11 avec Python 3.11. + +Les services d'Assemblyline utilisent les définitions d'étiquettes suivantes: + +| **Type d'étiquette** | **Description** | **Exemple d'étiquette** | +| :------------------: | :------------------------------------------------------------------------------------------------------------- | :------------------------: | +| dernière version | La version la plus récente (peut être instable). | `latest` | +| build_type | Type de construction utilisé. `dev` est la dernière version instable. `stable` est la dernière version stable. | `stable` ou `dev` | +| série | Détails de construction complets, comprenant la version et le type de build: `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | + +## Exécution de ce service + +Ce service est spécialement optimisé pour fonctionner dans le cadre d'un déploiement d'Assemblyline. + +Si vous souhaitez tester ce service localement, vous pouvez exécuter l'image Docker directement à partir d'un terminal: + + docker run \ + --name Suricata \ + --env SERVICE_API_HOST=http://`ip addr show docker0 | grep "inet " | awk '{print $2}' | cut -f1 -d"/"`:5003 \ + --network=host \ + cccs/assemblyline-service-suricata + +Pour ajouter ce service à votre déploiement d'Assemblyline, suivez ceci +[guide](https://cybercentrecanada.github.io/assemblyline4_docs/fr/developer_manual/services/run_your_service/#add-the-container-to-your-deployment). -Inside the container run: +## Documentation -```bash -python -m assemblyline_v4_service.dev.run_service_once suricata_.suricata_.Suricata /tmp/testing.pcap -``` +La documentation générale sur Assemblyline peut être consultée à l'adresse suivante: https://cybercentrecanada.github.io/assemblyline4_docs/ diff --git a/pipelines/.cruft.json b/pipelines/.cruft.json index 3025672..8135a03 100644 --- a/pipelines/.cruft.json +++ b/pipelines/.cruft.json @@ -7,7 +7,7 @@ "__directory_name": "pipelines", "classification": "UNCLASSIFIED", "is_public": true, - "test_in_container": false, + "test_in_container": true, "test_versions": "default", "__setupscript": "setup.bash", "_extensions": [ diff --git a/pipelines/azure-tests.yaml b/pipelines/azure-tests.yaml index 1932e12..7b4bdd3 100644 --- a/pipelines/azure-tests.yaml +++ b/pipelines/azure-tests.yaml @@ -28,5 +28,4 @@ extends: template: stages/test-service.yaml@PipelineTemplates parameters: samples_repo: unittest-samples - test_container: "false" - \ No newline at end of file + test_container: "true" diff --git a/pkglist.txt b/pkglist.txt new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d173d4d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.cruft] +skip = ["pkglist.txt", "README.md", "suricata", "tests"] diff --git a/requirements.txt b/requirements.txt index 213abfa..2e84945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ +assemblyline +assemblyline-v4-service assemblyline-service-utilities simplejson python-dateutil suricata-update suricataparser async_timeout -tenacity \ No newline at end of file +tenacity diff --git a/service_manifest.yml b/service_manifest.yml index 968fe47..505ceca 100644 --- a/service_manifest.yml +++ b/service_manifest.yml @@ -2,21 +2,26 @@ name: Suricata version: $SERVICE_TAG description: This service scans network capture files with signature and extract files from network capture. +# Regex defining the types of files the service accepts and rejects accepts: network/.* rejects: empty|metadata/.* +# At which stage the service should run (one of FILTER, EXTRACT, CORE, SECONDARY, POST, REVIEW) +# NOTE: Stages are executed in the order defined in the list stage: CORE +# Which category the service is part of (one of Antivirus, Dynamic Analysis, External, Extraction, Filtering, Internet Connected, Networking, Static Analysis) category: Networking +# Does the service require access to the file to perform its task +# If set to false, the service will only have access to the file metadata (e.g. Hashes, size, type, ...) file_required: true +# Maximum execution time the service has before it's considered to be timed out timeout: 180 -disable_cache: false +# is the service enabled by default enabled: true -is_external: false -licence_count: 0 -uses_temp_submission_data: true privileged: true +uses_temp_submission_data: true config: sure_score: ["MALWARE", "TROJAN", "CURRENT_EVENTS", "CnC", "Checkin"] diff --git a/tests/gentests.py b/tests/gentests.py new file mode 100755 index 0000000..7ae6dde --- /dev/null +++ b/tests/gentests.py @@ -0,0 +1,30 @@ +#!/bin/env python +import os + +from assemblyline.common.importing import load_module_by_path +from assemblyline_service_utilities.testing.helper import TestHelper + +cwd = os.getcwd() +# Force manifest location +os.environ["SERVICE_MANIFEST_PATH"] = os.path.join(cwd, "service_manifest.yml") + +# Setup folder locations +RESULTS_FOLDER = os.path.join(cwd, "tests", "results") +SAMPLES_FOLDER = os.path.join(cwd, "tests", "samples") + +# Find which module we're working on +module = os.environ.get("SERVICE_PATH") +if not module: + for line in open("Dockerfile", "r").readlines(): + if line.startswith("ENV SERVICE_PATH"): + module = line[17:].strip() + break + +# Initialize test helper +service_class = load_module_by_path(module, cwd) +if os.path.exists(SAMPLES_FOLDER): + th = TestHelper(service_class, RESULTS_FOLDER, SAMPLES_FOLDER) +else: + th = TestHelper(service_class, RESULTS_FOLDER) + +th.regenerate_results(save_files=False) diff --git a/tests/gentests.sh b/tests/gentests.sh new file mode 100755 index 0000000..2c9bb3c --- /dev/null +++ b/tests/gentests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +docker build \ + --pull \ + --build-arg branch=stable \ + -t ${PWD##*/}:gentests \ + -f ./Dockerfile \ + . + +if [[ -n "$FULL_SAMPLES_LOCATION" ]]; then + MOUNT_SAMPLES="-v ${FULL_SAMPLES_LOCATION}:/opt/samples" + ENV_SAMPLES="-e FULL_SAMPLES_LOCATION=/opt/samples" +fi +docker run \ + -t\ + --rm \ + -e FULL_SELF_LOCATION=/opt/al_service \ + $ENV_SAMPLES \ + -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla \ + -v $(pwd)/tests/:/opt/al_service/tests/ \ + $MOUNT_SAMPLES \ + ${PWD##*/}:gentests \ + bash -c "pip install -U -r tests/requirements.txt; python /opt/al_service/tests/gentests.py" diff --git a/tests/pytest.sh b/tests/pytest.sh new file mode 100755 index 0000000..66655c3 --- /dev/null +++ b/tests/pytest.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +docker build \ + --pull \ + --build-arg branch=stable \ + -t ${PWD##*/}:pytest \ + -f ./Dockerfile \ + . + +if [[ -n "$FULL_SAMPLES_LOCATION" ]]; then + MOUNT_SAMPLES="-v ${FULL_SAMPLES_LOCATION}:/opt/samples" + ENV_SAMPLES="-e FULL_SAMPLES_LOCATION=/opt/samples" +fi +docker run \ + -t \ + --rm \ + -e FULL_SELF_LOCATION=/opt/al_service \ + $ENV_SAMPLES \ + -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla \ + -v $(pwd)/tests/:/opt/al_service/tests/ \ + $MOUNT_SAMPLES \ + ${PWD##*/}:pytest \ + bash -c "pip install -U -r tests/requirements.txt; pytest -p no:cacheprovider --durations=10 -rsx -vv -x" diff --git a/tests/requirements.txt b/tests/requirements.txt index 2acab3c..6e3d947 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ +assemblyline assemblyline-service-utilities pytest From 962bfbb89a5f570e72300cdfaa815037acf26f3f Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:53:22 -0500 Subject: [PATCH 2/4] Update azure-tests.yaml --- pipelines/azure-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/azure-tests.yaml b/pipelines/azure-tests.yaml index 7b4bdd3..d04ae1a 100644 --- a/pipelines/azure-tests.yaml +++ b/pipelines/azure-tests.yaml @@ -29,3 +29,4 @@ extends: parameters: samples_repo: unittest-samples test_container: "true" + timeout: "20" From 78c400aaed9316c9baf4df776cce4fa92c78c06c Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:53:41 -0500 Subject: [PATCH 3/4] Update azure-build.yaml --- pipelines/azure-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/azure-build.yaml b/pipelines/azure-build.yaml index ee361e6..12316ca 100644 --- a/pipelines/azure-build.yaml +++ b/pipelines/azure-build.yaml @@ -31,6 +31,6 @@ extends: parameters: is_public: "true" samples_repo: unittest-samples - timeout: "15" + timeout: "20" labels: classification: "UNCLASSIFIED" From b298e92848958fba013bb4240f833411f5067c63 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:44:17 +0000 Subject: [PATCH 4/4] Update --- .cruft.json | 4 ++-- CONTRIBUTING.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.cruft.json b/.cruft.json index 7020d61..524c3f1 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/CybercentreCanada/assemblyline-service-template.git", - "commit": "6a32666117e30dbfb990bfaf6e7c9eb25badd03a", + "commit": "99becf8ed2990a4063a5bb1f8ce878824a5a1a7b", "checkout": null, "context": { "cookiecutter": { @@ -8,7 +8,7 @@ "__svc_name": "suricata", "__repository": "assemblyline-service-suricata", "__pkg_name": "suricata", - "__class_name": "Suricata", + "class_name": "Suricata", "short_description": "This service scans network capture files with signature and extract files from network capture.", "short_description_fr": "Ce service analyse les fichiers de capture réseau avec signature et extrait les fichiers de la capture réseau.", "stage": "CORE", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9785be..7b78163 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ We check for new pull requests every day and will merge them in once they have b # Guide de contribution d'Assemblyline -Ce guide couvre les bases de la façon de contribuer au projet Assemblyline. +Ce guide couvre les bases afin de contribuer au projet Assemblyline. Le code Python doit suivre les directives PEP8 définies ici: [Directives PEP8](https://www.python.org/dev/peps/pep-0008/).