diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index a4eb367..e2c8103 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -18,14 +18,29 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} + - uses: mamba-org/setup-micromamba@v1.8.1 - - name: Set up environment + - name: Fetch dependancies + run: sudo apt-get install libpcap-dev + + - name: Set up and scan environment run: echo $ENV_FILE | base64 --decode > manager/.env && make init + - name: CI-Specific optimization (Disable GPU) + run: sed 's/ - torch==/ - --extra-index-url https:\/\/download.pytorch.org\/whl\/cpu\n&/' -i manager/environment-manager.yml + + - name: CI-Specific optimization (Remove Development Envs) + run: | + micromamba env remove -n attacker-dev + micromamba env remove -n honeypot-dev + micromamba env remove -n manager-dev + + - name: CI-Specific optimization (Purge Cache) + run: pip cache purge && micromamba clean -a + + - name: CI-Specific optimization (Lower SIEM message journal max size) + run: echo "RUN sed -i 's/#message_journal_max_size = 5gb/message_journal_max_size = 500mb/' /usr/share/graylog/data/config/graylog.conf" >> siem/Dockerfile + - name: Build Containers run: make build diff --git a/.gitignore b/.gitignore index 293d62d..04f583c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,10 @@ coverage.xml db.sqlite3 docker-compose-local-overrides.yaml pcaps/ -venv \ No newline at end of file +venv +reqs* +*/reqs* + +enviroment.yml + +*/environment.yml \ No newline at end of file diff --git a/.safety-check-policy.yml b/.safety-check-policy.yml new file mode 100644 index 0000000..0e64dbc --- /dev/null +++ b/.safety-check-policy.yml @@ -0,0 +1,8 @@ +security: + continue-on-vulnerability-error: False + ignore-cvss-severity-below: 0 + ignore-cvss-unknown-severity: False + ignore-vulnerabilities: + 65052: + reason: Disputed + not impacted accd to vendor + expires: '2024-06-17' diff --git a/.yamllint b/.yamllint index c2bd63c..8932288 100644 --- a/.yamllint +++ b/.yamllint @@ -5,7 +5,8 @@ extends: default ignore: | aica-manager/ target/etc/aica-config.yml + */environment.yml rules: line-length: - max: 120 + max: 200 diff --git a/Makefile b/Makefile index f483fb9..b47808f 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,83 @@ export DOCKER_SCAN_SUGGEST := false export DOCKER_BUILDKIT := 1 -CONDA := conda run --no-capture-output -n aica +MAMBA_RUN := ${MAMBA_EXE} run -n aica-make check-env: ifndef MODE $(error MODE is undefined) endif -init: environment.yml - @conda env update -f environment.yml + +init-core-env: + @${MAMBA_EXE} -y env create -f environment-core.yml + @${MAMBA_RUN} python3 compute_dev.py + +init-dev-envs: + @${MAMBA_EXE} -y env create -f attacker/environment.yml + @${MAMBA_EXE} -y env create -f honeypot/environment.yml + @${MAMBA_EXE} -y env create -f manager/environment.yml + +security-precheck-init: + @${MAMBA_EXE} -y env create -f environment-security.yml + @${MAMBA_EXE} run -n aica-secprecheck python3 compute_security.py + +security-precheck-bandit: + @${MAMBA_EXE} run -n aica-secprecheck bandit -q -ll -ii -r manager/ + +security-precheck-safety-core: + @${MAMBA_EXE} run -n aica-secprecheck safety check -r reqs.txt --policy-file .safety-check-policy.yml + +security-precheck-safety-attacker: + @${MAMBA_EXE} run -n aica-secprecheck safety check -r attacker/reqs.txt --policy-file .safety-check-policy.yml + +security-precheck-safety-honeypot: + @${MAMBA_EXE} run -n aica-secprecheck safety check -r honeypot/reqs.txt --policy-file .safety-check-policy.yml + +security-precheck-safety-manager: + @${MAMBA_EXE} run -n aica-secprecheck safety check -r manager/reqs.txt --policy-file .safety-check-policy.yml + +security-precheck: security-precheck-init security-precheck-bandit security-precheck-safety-core security-precheck-safety-attacker security-precheck-safety-honeypot security-precheck-safety-manager + + + +security-postcheck: + @${MAMBA_EXE} list -n manager-dev --json | (${MAMBA_RUN} jake ddt -t CONDA_JSON) + @${MAMBA_EXE} list -n honeypot-dev --json | (${MAMBA_RUN} jake ddt -t CONDA_JSON) + @${MAMBA_EXE} list -n attacker-dev --json | (${MAMBA_RUN} jake ddt -t CONDA_JSON) + + +init: security-precheck init-core-env init-dev-envs security-postcheck + + black: - @${CONDA} black -q manager/ attacker/ + @${MAMBA_RUN} black -q manager/ attacker/ lint: - @${CONDA} yamllint . - @${CONDA} bashlint . - @${CONDA} black --check --diff -q manager/ attacker/ - @MYPYPATH=manager ${CONDA} mypy --install-types --warn-unreachable --strict --non-interactive --exclude test manager/ + @${MAMBA_RUN} yamllint . + @${MAMBA_RUN} bashlint . + @${MAMBA_RUN} black --check --diff -q manager/ attacker/ + @${MAMBA_RUN} mypy --install-types --warn-unreachable --strict --non-interactive --exclude test manager/ + + -security: - @${CONDA} bandit -q -ll -ii -r manager/ - @${CONDA} safety check -r manager/requirements.txt - @${CONDA} safety check -r honeypot/requirements.txt build: check-env - @docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml build + @docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml build -test: lint security - @MODE=emu docker compose -f docker-compose.yml -f docker-compose-emu.yml \ - run -e SKIP_TASKS=true --rm \ +tests: + @MODE=emu docker compose -f docker-compose.yml -f docker-compose-emu.yml up --wait -d && \ + docker exec -e SKIP_TASKS=true \ manager /bin/bash -c " \ - /opt/venv/bin/coverage run --omit='*test*' manage.py test --noinput && \ - /opt/venv/bin/coverage report --fail-under=30" + /usr/src/app/bin/micromamba run -n base coverage run --omit='*test*' manage.py test --noinput && \ + /usr/src/app/bin/micromamba run -n base coverage report --fail-under=30" + +test-initless: lint security-precheck security-postcheck tests + + +test: lint tests + start: check-env @docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml up --wait -d diff --git a/README.md b/README.md index d04b928..0e64fd2 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ This project will build on the ideas of the AICA framework as outlined in [Thero It is important to ensure your main branch is up-to-date before each working session, and you should commit your changes incrementally and often to ensure minimal divergence and chance of merge conflicts. Changes should be "intact" functionally (i.e., don't submit partially-completed work) and keep the main repository in a working state. This means you should think about functionality in the smallest possible chunks to keep your contributed work up to date. -You can bootstrap your environment with `make deps`, which will create a conda environment for building and testing. +You can bootstrap your environment with `make init`, which will create a conda environment for building and testing. Changes must be pushed to a branch and PR'ed to main. Before pushing your changes, you should first locally execute a `make test` and ensure it completes successfully. If it does not, either fix the issues or propose exclusions to the relevant test areas (will be subject to peer review). diff --git a/attacker/Dockerfile b/attacker/Dockerfile index 0b10cf7..3196a3d 100644 --- a/attacker/Dockerfile +++ b/attacker/Dockerfile @@ -17,21 +17,27 @@ RUN apt-get update && \ openssh-server \ python3-dev \ python3-pip \ - python3-venv + python3-venv \ + curl RUN ssh-keygen -A COPY docker_entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker_entrypoint.sh -RUN useradd kali + + +RUN useradd -m kali USER kali + WORKDIR /home/kali -COPY requirements.txt . -RUN python3 -m venv attacker && \ - ./attacker/bin/python -m pip install -Ur requirements.txt +#Install micromamba +RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba +COPY environment-attacker.yml . +RUN bin/micromamba env create -f environment-attacker.yml + COPY tests/ ./tests diff --git a/attacker/environment-attacker.yml b/attacker/environment-attacker.yml new file mode 100644 index 0000000..f9426ab --- /dev/null +++ b/attacker/environment-attacker.yml @@ -0,0 +1,11 @@ +--- +name: base +channels: + - conda-forge +dependencies: + - python=3.11.9 + - pip=24.* + - PyJWT=2.8.0 + - python-dotenv=1.0.1 + - requests=2.31.0 + - selenium=4.18.1 diff --git a/attacker/requirements.txt b/attacker/requirements.txt deleted file mode 100644 index 25326cc..0000000 --- a/attacker/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -PyJWT==2.8.0 -python-dotenv==1.0.1 -requests==2.31.0 -selenium==4.18.1 \ No newline at end of file diff --git a/compute_dev.py b/compute_dev.py new file mode 100644 index 0000000..0a9e3d2 --- /dev/null +++ b/compute_dev.py @@ -0,0 +1,56 @@ +import yaml +from packaging.version import Version + +def pkg_list_to_dict(depsList: list[str], isPip: bool) -> dict: + depDict = {} + for package in depsList: + if not isPip and type(package) is not str: + continue + pkgName, pkgVer = package.split("==" if isPip else "=") + depDict[pkgName] = pkgVer + return depDict + +def dev_corrections(pipDict: dict) -> None: + if "psycopg2" in pipDict.keys(): + ver = pipDict["psycopg2"] + del pipDict["psycopg2"] + pipDict["psycopg2-binary"] = ver + + + + +def main() -> None: + for reqLoc in ["attacker", "honeypot", "manager"]: + with open(reqLoc + "/environment-"+ reqLoc + ".yml", "r") as baseEnvFp: + handle : dict = yaml.load(baseEnvFp, yaml.Loader) + handle["name"] = reqLoc + "-dev" + + pipDeps = {} + print(type(handle["dependencies"][-1])) + if type(handle["dependencies"][-1]) is dict and "pip" in handle["dependencies"][-1].keys(): + pipDeps :dict = pkg_list_to_dict(handle["dependencies"][-1]["pip"], True) + dev_corrections(pipDeps) + + deps :dict = pkg_list_to_dict(handle["dependencies"], False) + print(deps) + dev_corrections(deps) + handle["dependencies"]=[dep + "=" + deps[dep] for dep in deps.keys()] + + if len(pipDeps) != 0: + parsedPip = {"pip":[dep + "==" + pipDeps[dep] for dep in pipDeps.keys()]} + if "torch" in pipDeps.keys(): + torchloc = parsedPip["pip"].index("torch=="+ pipDeps["torch"]) + parsedPip["pip"].insert(torchloc, "--extra-index-url https://download.pytorch.org/whl/cpu") + + handle["dependencies"].append(parsedPip) + + + with open(reqLoc+"/environment.yml", "w") as computedEnvFp: + computedEnvFp.write("---\n") + yaml.dump(handle, computedEnvFp) + + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/compute_security.py b/compute_security.py new file mode 100644 index 0000000..0af3738 --- /dev/null +++ b/compute_security.py @@ -0,0 +1,55 @@ +#In a seperate file for now + +import yaml + +def quick_figure(package: str, ver): + return package if ver == -1 else package + "==" + ver + +def dev_alt(altList:list, pkg: str) -> list: + pkgName, pkgVer = pkg.split("==") if len(pkg.split("==")) == 2 else [pkg, -1] + if pkgName == "psycopg2": + altList.append(quick_figure("psycopg2-binary",pkgVer)) + elif pkgName == "torch": + altList.append("-i https://download.pytorch.org/whl/cpu") + altList.append(pkg) + + +def transcribe(envLoc: str, envName: str): + with open(envLoc + "environment-"+ envName + ".yml", "r") as baseEnvFp: + normalDeps=[] + devDeps=[] + + condaDeps : list = yaml.load(baseEnvFp, yaml.Loader)["dependencies"] + for dep in condaDeps[:-1]: + dep = dep.replace("=", "==") + + normalDeps.append(dep) + dev_alt(devDeps, dep) + + + if type(condaDeps[-1]) is dict and "pip" in condaDeps[-1].keys(): + for dep in condaDeps[-1]["pip"]: + normalDeps.append(dep) + dev_alt(devDeps, dep) + + #FIXME: Sometimes this doesn't update existing req files + with open(envLoc+"reqs.txt", "w") as computedReqs: + for line in normalDeps: + computedReqs.write(line + "\n") + + print(devDeps) + with open(envLoc+"reqsDev.txt", "w") as computedDevReqs: + for line in devDeps: + computedDevReqs.write(line + "\n") + +def main() -> None: + envList = [[a+"/",a] for a in ["attacker", "honeypot", "manager"]] + envList.append(["","core"]) + + for r in envList: + transcribe(*r) + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/computed-devel-environment.yml b/computed-devel-environment.yml new file mode 100644 index 0000000..a6fe8d8 --- /dev/null +++ b/computed-devel-environment.yml @@ -0,0 +1,67 @@ +--- +channels: + - conda-forge +dependencies: + - python=3.11 + - mypy=1.8.* + - yamllint=1.35.* + - pip=24.* + - pip: + - bandit==1.7.7 + - bashlint==0.1.1 + - black==24.2.0 + - celery-types==0.22.0 + - charset-normalizer==3.3.2 + - django-stubs==4.2.7 + - networkx-stubs==0.0.1 + - pandas-stubs==2.2.0.240218 + - safety==3.0.1 + - types-requests==2.31.0.20240218 + - types-dateparser==1.1.4.20240106 + - types-openpyxl==3.1.0.20240220 + - types-paramiko==3.4.0.20240103 + - urllib3==2.2.1 + - Django==4.2.11 + - amqp==5.2.0 + - backoff==2.2.1 + - celery==5.3.6 + - coverage==7.4.4 + - dateparser==1.2.0 + - django-extensions==3.2.3 + - fasteners==0.19 + - flower==2.0.1 + - google-re2==1.1 + - gunicorn==21.2.0 + - ipwhois==1.2.0 + - joblib==1.4.0 + - kombu==5.3.7 + - mac-vendor-lookup==0.1.12 + - neo4j==5.19.0 + - netaddr==1.2.1 + - netflow==0.12.2 + - netifaces2==0.0.21 + - networkx==3.3 + - pandas==2.2.2 + - paramiko==3.4.0 + - pecanpy==2.0.9 + - pymongo==4.6.3 + - pyshark==0.6 + - python-dotenv==1.0.1 + - python3-nmap==1.6.0 + - pyyaml==6.0.1 + - requests==2.21.0 + - scikit-learn==1.4.2 + - sqlalchemy==2.0.29 + - stix2==3.0.1 + - supervisor-stdout==0.1.1 + - torch==2.3.0 + - torch-geometric==2.5.3 + - watchdog==4.0.0 + - vt-py==0.18.0 + - scapy==2.5.0 + - pcapyplus==1.0.1 + - opencanary==0.9.2 + - PyJWT==2.8.0 + - selenium==4.18.1 + - psycopg2-binary==2.9.9 +name: aica-devel diff --git a/docker-compose-emu.yml b/docker-compose-emu.yml index 7ac847a..86e91fe 100644 --- a/docker-compose-emu.yml +++ b/docker-compose-emu.yml @@ -71,7 +71,7 @@ services: honeypot: build: honeypot container_name: honeypot - command: bash -c "source canary/bin/activate && /canary/bin/opencanaryd --dev" + command: bash -c "/usr/local/bin/_entrypoint.sh && /opt/conda/bin/opencanaryd --dev" volumes: - ./honeypot/opencanary.conf:/etc/opencanaryd/opencanary.conf networks: diff --git a/docker-compose.yml b/docker-compose.yml index 6ef7d9a..d2991e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: manager: build: manager container_name: manager - restart: unless-stopped + restart: on-failure:2 #unless-stopped env_file: - manager/.env ports: diff --git a/environment.yml b/environment-core.yml similarity index 85% rename from environment.yml rename to environment-core.yml index 3d85cda..ff34a2f 100644 --- a/environment.yml +++ b/environment-core.yml @@ -1,17 +1,19 @@ --- -name: aica +name: aica-make channels: - conda-forge dependencies: - - python=3.11 + - python=3.11.9 - mypy=1.8.* - yamllint=1.35.* - pip=24.* + - pyyaml + - packaging # Some things are down here for better cross-platform compatibility - pip: - bandit==1.7.7 - bashlint==0.1.1 - - black==24.2.0 + - black==24.3.0 - celery-types==0.22.0 - charset-normalizer==3.3.2 - django-stubs==4.2.7 @@ -23,3 +25,4 @@ dependencies: - types-openpyxl==3.1.0.20240220 - types-paramiko==3.4.0.20240103 - urllib3==2.2.1 + - jake diff --git a/environment-security.yml b/environment-security.yml new file mode 100644 index 0000000..133f006 --- /dev/null +++ b/environment-security.yml @@ -0,0 +1,13 @@ +--- +name: aica-secprecheck +channels: + - conda-forge +dependencies: #MUST BE IDENTICAL VERSIONS TO CORE PACKAGE + - python=3.11.9 + - pip=24.* + - pyyaml + # Some things are down here for better cross-platform compatibility + - pip: + - bandit==1.7.7 + - safety==3.0.1 + - jake diff --git a/honeypot/Dockerfile b/honeypot/Dockerfile index 51957e5..8e89b22 100644 --- a/honeypot/Dockerfile +++ b/honeypot/Dockerfile @@ -1,12 +1,20 @@ -FROM alpine:3.19 +FROM mambaorg/micromamba:alpine3.19 +#FROM alpine:3.19 + +USER root ENV CRYPTOGRAPHY_DONT_BUILD_RUST 1 RUN apk update && apk upgrade && \ - apk add git python3 py3-pip python3-dev alpine-sdk openssl-dev libpcap-dev libffi-dev bash sudo iptables + apk add git python3 py3-pip python3-dev alpine-sdk openssl-dev libpcap-dev libffi-dev bash sudo iptables +COPY opencanary.conf /etc/opencanaryd/opencanary.conf + -COPY requirements.txt . -RUN python3 -m venv canary && \ - bash -c "source canary/bin/activate && pip install -Ur requirements.txt" +USER $MAMBA_USER +COPY --chown=$MAMBA_USER:$MAMBA_USER environment-honeypot.yml /tmp/env.yml +#COPY --chown=root:root environment-honeypot.yml /tmp/env.yml +RUN micromamba install -y -n base -f /tmp/env.yml && \ + micromamba clean --all --yes -COPY opencanary.conf /etc/opencanaryd/opencanary.conf \ No newline at end of file +USER root +#ENTRYPOINT ["/usr/local/bin/_entrypoint.sh", "/opt/conda/bin/opencanaryd", "--dev"] \ No newline at end of file diff --git a/honeypot/environment-honeypot.yml b/honeypot/environment-honeypot.yml new file mode 100644 index 0000000..07e6015 --- /dev/null +++ b/honeypot/environment-honeypot.yml @@ -0,0 +1,11 @@ +--- +name: base +channels: + - conda-forge +dependencies: + - python=3.11.9 + - pip=24.* + - scapy=2.5.0 + - pip: + - pcapyplus==1.0.1 + - opencanary==0.9.2 diff --git a/honeypot/requirements.txt b/honeypot/requirements.txt deleted file mode 100644 index f82f05d..0000000 --- a/honeypot/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -scapy==2.5.0 -pcapyplus==1.0.1 -opencanary==0.9.2 \ No newline at end of file diff --git a/ids/suricata.yaml b/ids/suricata.yaml index 0b0c7f7..cf83a40 100644 --- a/ids/suricata.yaml +++ b/ids/suricata.yaml @@ -362,7 +362,7 @@ logging: # compiled with the --enable-debug configure option. # # This value is overridden by the SC_LOG_LEVEL env var. - default-log-level: falsetice + default-log-level: info # The default output format. Optional parameter, should default to # something reasonable if not provided. Can be overridden in an diff --git a/manager/Dockerfile b/manager/Dockerfile index 7e6dbdf..d1fffcd 100644 --- a/manager/Dockerfile +++ b/manager/Dockerfile @@ -38,15 +38,13 @@ RUN --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ tcpdump \ tshark \ net-tools \ - vim + vim \ + curl -COPY requirements.txt . - -RUN python3 -m venv /opt/venv && \ - /opt/venv/bin/pip3 --no-cache install -qU pip - -RUN python3 -m venv /opt/venv && \ - /opt/venv/bin/pip3 --no-cache install -qUr requirements.txt +#Install micromamba +RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba +COPY environment-manager.yml . +RUN bin/micromamba env create -f environment-manager.yml # Install dependencies RUN apt-get -y remove \ @@ -75,5 +73,5 @@ COPY . . COPY docker_entrypoint.sh /usr/local/bin/docker_entrypoint.sh RUN chmod +x /usr/local/bin/docker_entrypoint.sh -CMD /usr/local/bin/docker_entrypoint.sh +CMD ["bin/micromamba", "run", "-n", "base", "/usr/local/bin/docker_entrypoint.sh"] diff --git a/manager/docker_entrypoint.sh b/manager/docker_entrypoint.sh index 625829e..a881363 100755 --- a/manager/docker_entrypoint.sh +++ b/manager/docker_entrypoint.sh @@ -1,4 +1,8 @@ #!/bin/bash +# echo "Activating micromamba enviroment" +# eval "$(/usr/src/app/bin/micromamba shell hook --shell bash )" +# /usr/src/app/bin/micromamba activate base + set -e cd /usr/src/app @@ -22,18 +26,20 @@ service supervisor start # Tell Celery to not run tasks on the following manage.py invocations export SKIP_TASKS=true + + # Apply database migrations echo "Creating Django Database" -/opt/venv/bin/python3 manage.py sqlcreate -D \ +python3 manage.py sqlcreate -D \ | grep -v USER \ - | /opt/venv/bin/python3 manage.py dbshell --database postgres + | python3 manage.py dbshell --database postgres -/opt/venv/bin/python3 manage.py makemigrations && /opt/venv/bin/python3 manage.py migrate +python3 manage.py makemigrations && python3 manage.py migrate # Create superuser for Django if [ "$DJANGO_SUPERUSER_USERNAME" ]; then echo "Creating Django Superuser" - /opt/venv/bin/python3 manage.py createsuperuser --noinput + python3 manage.py createsuperuser --noinput fi echo "Manager started in mode: ${MODE}" diff --git a/manager/environment-manager.yml b/manager/environment-manager.yml new file mode 100644 index 0000000..754bae1 --- /dev/null +++ b/manager/environment-manager.yml @@ -0,0 +1,46 @@ +--- +name: base +channels: + - conda-forge +dependencies: + - python=3.11.9 + - pip=24.* + - Django=4.2.13 + - amqp=5.2.0 + - backoff=2.2.1 + - celery=5.3.6 + - coverage=7.4.4 + - dateparser=1.2.0 + - django-extensions=3.2.3 + - flower=2.0.1 + - google-re2=1.1 + - gunicorn=21.2.0 + - joblib=1.4.0 + - kombu=5.3.7 + - networkx=3.3 + - pandas=2.2.2 + - paramiko=3.4.0 + - psycopg2=2.9.9 + - pymongo=4.6.3 + - pyshark=0.6 + - python-dotenv=1.0.1 + - pyyaml=6.0.1 + - requests=2.31.0 + - scikit-learn=1.4.2 + - sqlalchemy=2.0.29 + - watchdog=4.0.0 + - vt-py=0.18.0 + - pip: + - fasteners==0.19 + - ipwhois==1.2.0 + - mac-vendor-lookup==0.1.12 + - neo4j==5.19.0 + - netaddr==1.2.1 + - netflow==0.12.2 + - netifaces2==0.0.21 + - pecanpy==2.0.9 + - python3-nmap==1.6.0 + - stix2==3.0.1 + - supervisor-stdout==0.1.1 + - torch-geometric==2.5.3 + - torch==2.3.0 diff --git a/manager/requirements.txt b/manager/requirements.txt deleted file mode 100644 index 60d3147..0000000 --- a/manager/requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -Django==4.2.11 -amqp==5.2.0 -backoff==2.2.1 -celery==5.3.6 -coverage==7.4.4 -dateparser==1.2.0 -django-extensions==3.2.3 -fasteners==0.19 -flower==2.0.1 -google-re2==1.1 -gunicorn==21.2.0 -ipwhois==1.2.0 -joblib==1.4.0 -kombu==5.3.7 -mac-vendor-lookup==0.1.12 -neo4j==5.19.0 -netaddr==1.2.1 -netflow==0.12.2 -netifaces2==0.0.21 -networkx==3.3 -pandas==2.2.2 -paramiko==3.4.0 -pecanpy==2.0.9 -psycopg2==2.9.9 -pymongo==4.6.3 -pyshark==0.6 -python-dotenv==1.0.1 -python3-nmap==1.6.0 -pyyaml==6.0.1 -requests==2.31.0 -scikit-learn==1.4.2 -sqlalchemy==2.0.29 -stix2==3.0.1 -supervisor-stdout==0.1.1 -torch==2.3.0 -torch-geometric==2.5.3 -watchdog==4.0.0 -vt-py==0.18.0 \ No newline at end of file diff --git a/shvvl.py b/shvvl.py new file mode 100644 index 0000000..f932c8a --- /dev/null +++ b/shvvl.py @@ -0,0 +1,36 @@ +import hashlib + +def shvvl(tag: str, bpf: int) -> bytes: + ''' + This is SHVVL. The only important thing is that the "type" of node is the first string in the tag + ''' + + sectors = tag.split("\0") + typehash = hashlib.md5(bytes(sectors[0], "UTF8"), usedforsecurity=False).digest() + + out = bytearray() + hashfunc = hashlib.new('shake_256', usedforsecurity=False) + for sector in sectors: + hashfunc = hashlib.new('shake_256', usedforsecurity=False) + + blockInput = bytearray(sector, "UTF8") + blockInput = blockInput + typehash + hashfunc.update(blockInput) + out += hashfunc.digest(bpf) + + print(out.hex()) + return out + +def shvvl_float(tag: str, bpf: int) -> list[float]: + out = list() + for bite in shvvl(tag, bpf): + for l in range(8): + out.append(1.0 if (bite&(1<