Prieto conda (resubmission) (#261)

* Created python script to dynamically compute devel enviroment file

* Added SHVVL

* Added conda to attacker and hp

* Added dev env for manager

* Added jake

* Fixed honeypot, large change. Also altered linting behaviour.

* Got manager running

* Re-enabled yamllint, and fixed entry for honeypot in compose

* Made CI mamba

* Modified workflow

* Fixed makefile errors

* Saved space on dev env

* Security + cleanup

* Modified security policy

* Made safety not mad

* CI opti

* CI Correction

* CI Correction pt 2

* CI + Gitignore fixes

* Make linter happy

* More fixes

* Loosened yamllint rules

* makefile

* Added additional opti

* workflow

.github/workflows/makefile.yml

@@ -18,14 +18,29 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          python-version: ${{ matrix.python-version }}
+      - uses: mamba-org/[email protected]
 
-      - name: Set up environment
+      - name: Fetch dependancies
+        run: sudo apt-get install libpcap-dev
+
+      - name: Set up and scan environment
         run: echo $ENV_FILE | base64 --decode > manager/.env && make init
 
+      - name: CI-Specific optimization (Disable GPU)
+        run: sed 's/      - torch==/      - --extra-index-url https:\/\/download.pytorch.org\/whl\/cpu\n&/' -i manager/environment-manager.yml
+
+      - name: CI-Specific optimization (Remove Development Envs)
+        run: |
+          micromamba env remove -n attacker-dev
+          micromamba env remove -n honeypot-dev
+          micromamba env remove -n manager-dev
+
+      - name: CI-Specific optimization (Purge Cache)
+        run: pip cache purge && micromamba clean -a
+
+      - name: CI-Specific optimization (Lower SIEM message journal max size)
+        run: echo "RUN sed -i 's/#message_journal_max_size = 5gb/message_journal_max_size = 500mb/' /usr/share/graylog/data/config/graylog.conf" >> siem/Dockerfile
+
       - name: Build Containers
         run: make build
 

.gitignore

@@ -10,4 +10,10 @@ coverage.xml
 db.sqlite3
 docker-compose-local-overrides.yaml
 pcaps/
-venv
+venv
+reqs*
+*/reqs*
+
+enviroment.yml
+
+*/environment.yml

.safety-check-policy.yml

@@ -0,0 +1,8 @@
+security:
+  continue-on-vulnerability-error: False
+  ignore-cvss-severity-below: 0
+  ignore-cvss-unknown-severity: False
+  ignore-vulnerabilities:
+    65052:
+      reason: Disputed + not impacted accd to vendor
+      expires: '2024-06-17'

.yamllint

@@ -5,7 +5,8 @@ extends: default
 ignore: |
   aica-manager/
   target/etc/aica-config.yml
+  */environment.yml
 
 rules:
   line-length:
-    max: 120
+    max: 200

Makefile

@@ -1,39 +1,83 @@
 export DOCKER_SCAN_SUGGEST := false
 export DOCKER_BUILDKIT := 1
 
-CONDA := conda run --no-capture-output -n aica
+MAMBA_RUN := ${MAMBA_EXE} run -n aica-make
 
 check-env:
 ifndef MODE
 		$(error MODE is undefined)
 endif
 
-init: environment.yml
-		@conda env update -f environment.yml
+
+init-core-env:
+		@${MAMBA_EXE} -y env create -f environment-core.yml
+		@${MAMBA_RUN} python3 compute_dev.py
+
+init-dev-envs:
+		@${MAMBA_EXE} -y env create -f attacker/environment.yml
+		@${MAMBA_EXE} -y env create -f honeypot/environment.yml
+		@${MAMBA_EXE} -y env create -f manager/environment.yml
+
+security-precheck-init:
+		@${MAMBA_EXE} -y env create -f environment-security.yml
+		@${MAMBA_EXE} run -n aica-secprecheck python3 compute_security.py
+
+security-precheck-bandit:
+		@${MAMBA_EXE} run -n aica-secprecheck bandit -q -ll -ii -r manager/
+
+security-precheck-safety-core:
+		@${MAMBA_EXE} run -n aica-secprecheck safety check -r reqs.txt --policy-file .safety-check-policy.yml
+
+security-precheck-safety-attacker:
+		@${MAMBA_EXE} run -n aica-secprecheck safety check -r attacker/reqs.txt --policy-file .safety-check-policy.yml
+
+security-precheck-safety-honeypot:
+		@${MAMBA_EXE} run -n aica-secprecheck safety check -r honeypot/reqs.txt --policy-file .safety-check-policy.yml
+
+security-precheck-safety-manager:
+		@${MAMBA_EXE} run -n aica-secprecheck safety check -r manager/reqs.txt --policy-file .safety-check-policy.yml
+
+security-precheck: security-precheck-init security-precheck-bandit security-precheck-safety-core security-precheck-safety-attacker security-precheck-safety-honeypot security-precheck-safety-manager
+
+
+
+security-postcheck:
+		@${MAMBA_EXE} list -n manager-dev --json | (${MAMBA_RUN} jake ddt -t CONDA_JSON)
+		@${MAMBA_EXE} list -n honeypot-dev --json | (${MAMBA_RUN} jake ddt -t CONDA_JSON)
+		@${MAMBA_EXE} list -n attacker-dev --json | (${MAMBA_RUN} jake ddt -t CONDA_JSON)
+
+
+init: security-precheck init-core-env init-dev-envs security-postcheck
+
+
 
 black:
-		@${CONDA} black -q manager/ attacker/
+		@${MAMBA_RUN} black -q manager/ attacker/
 
 lint:
-		@${CONDA} yamllint .
-		@${CONDA} bashlint .
-		@${CONDA} black --check --diff -q manager/ attacker/
-		@MYPYPATH=manager ${CONDA} mypy --install-types --warn-unreachable --strict --non-interactive --exclude test manager/
+		@${MAMBA_RUN} yamllint .
+		@${MAMBA_RUN} bashlint .
+		@${MAMBA_RUN} black --check --diff -q manager/ attacker/
+		@${MAMBA_RUN} mypy --install-types --warn-unreachable --strict --non-interactive --exclude test manager/
+
+
 
-security:
-		@${CONDA} bandit -q -ll -ii -r manager/
-		@${CONDA} safety check -r manager/requirements.txt
-		@${CONDA} safety check -r honeypot/requirements.txt
 
 build: check-env
-		@docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml build
+		@docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml build 
 
-test: lint security
-		@MODE=emu docker compose -f docker-compose.yml -f docker-compose-emu.yml \
-			run -e SKIP_TASKS=true --rm \
+tests:
+		@MODE=emu docker compose -f docker-compose.yml -f docker-compose-emu.yml up --wait -d && \
+			docker exec -e SKIP_TASKS=true \
 			manager /bin/bash -c " \
-				/opt/venv/bin/coverage run --omit='*test*' manage.py test --noinput && \
-				/opt/venv/bin/coverage report --fail-under=30"
+				/usr/src/app/bin/micromamba run -n base coverage run --omit='*test*' manage.py test --noinput && \
+				/usr/src/app/bin/micromamba run -n base coverage report --fail-under=30"
+
+test-initless: lint security-precheck security-postcheck tests
+
+
+test: lint tests
+
 
 start: check-env 
 		@docker compose -f docker-compose.yml -f docker-compose-${MODE}.yml up --wait -d

README.md

@@ -33,7 +33,7 @@ This project will build on the ideas of the AICA framework as outlined in [Thero
 
 It is important to ensure your main branch is up-to-date before each working session, and you should commit your changes incrementally and often to ensure minimal divergence and chance of merge conflicts. Changes should be "intact" functionally (i.e., don't submit partially-completed work) and keep the main repository in a working state. This means you should think about functionality in the smallest possible chunks to keep your contributed work up to date.
 
-You can bootstrap your environment with `make deps`, which will create a conda environment for building and testing. 
+You can bootstrap your environment with `make init`, which will create a conda environment for building and testing. 
 
 Changes must be pushed to a branch and PR'ed to main. Before pushing your changes, you should first locally execute a `make test` and ensure it completes successfully. If it does not, either fix the issues or propose exclusions to the relevant test areas (will be subject to peer review).
 

attacker/Dockerfile

@@ -17,21 +17,27 @@ RUN apt-get update && \
         openssh-server \
         python3-dev \
         python3-pip \
-        python3-venv
+        python3-venv \
+        curl
 
 RUN ssh-keygen -A
 
 COPY docker_entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker_entrypoint.sh
 
-RUN useradd kali
+
+
+RUN useradd -m kali
 
 USER kali
 
+
 WORKDIR /home/kali
 
-COPY requirements.txt .
-RUN python3 -m venv attacker && \
-    ./attacker/bin/python -m pip install -Ur requirements.txt
+#Install micromamba
+RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+COPY environment-attacker.yml .
+RUN bin/micromamba env create -f environment-attacker.yml
+
 
 COPY tests/ ./tests

attacker/environment-attacker.yml

@@ -0,0 +1,11 @@
+---
+name: base
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11.9
+  - pip=24.*
+  - PyJWT=2.8.0
+  - python-dotenv=1.0.1
+  - requests=2.31.0
+  - selenium=4.18.1

compute_dev.py

@@ -0,0 +1,56 @@
+import yaml
+from packaging.version import Version
+
+def pkg_list_to_dict(depsList: list[str], isPip: bool) -> dict:
+    depDict = {}
+    for package in depsList:
+            if not isPip and type(package) is not str:
+                continue
+            pkgName, pkgVer = package.split("==" if isPip else "=")
+            depDict[pkgName] = pkgVer
+    return depDict
+
+def dev_corrections(pipDict: dict) -> None:
+    if "psycopg2" in pipDict.keys():
+        ver = pipDict["psycopg2"]
+        del pipDict["psycopg2"]
+        pipDict["psycopg2-binary"] = ver
+
+
+
+
+def main() -> None:
+    for reqLoc in ["attacker", "honeypot", "manager"]:
+        with open(reqLoc + "/environment-"+ reqLoc + ".yml", "r") as baseEnvFp:
+            handle : dict = yaml.load(baseEnvFp, yaml.Loader)
+            handle["name"] = reqLoc + "-dev"
+
+            pipDeps = {}
+            print(type(handle["dependencies"][-1]))
+            if type(handle["dependencies"][-1]) is dict and "pip" in handle["dependencies"][-1].keys():
+                pipDeps :dict =  pkg_list_to_dict(handle["dependencies"][-1]["pip"], True)
+                dev_corrections(pipDeps)
+
+            deps :dict =  pkg_list_to_dict(handle["dependencies"], False)
+            print(deps)
+            dev_corrections(deps)
+            handle["dependencies"]=[dep + "=" + deps[dep] for dep in deps.keys()]
+
+            if len(pipDeps) != 0:
+                parsedPip = {"pip":[dep + "==" + pipDeps[dep] for dep in pipDeps.keys()]}
+                if "torch" in pipDeps.keys():
+                    torchloc = parsedPip["pip"].index("torch=="+ pipDeps["torch"])
+                    parsedPip["pip"].insert(torchloc, "--extra-index-url https://download.pytorch.org/whl/cpu")
+
+                handle["dependencies"].append(parsedPip)
+
+
+            with open(reqLoc+"/environment.yml", "w") as computedEnvFp:
+                computedEnvFp.write("---\n")
+                yaml.dump(handle, computedEnvFp)
+
+
+
+
+if __name__ == "__main__":
+    main()

compute_security.py

@@ -0,0 +1,55 @@
+#In a seperate file for now
+
+import yaml
+
+def quick_figure(package: str, ver):
+    return package if ver == -1 else package + "==" + ver
+
+def dev_alt(altList:list, pkg: str) -> list:
+    pkgName, pkgVer = pkg.split("==") if len(pkg.split("==")) == 2 else [pkg, -1]
+    if pkgName == "psycopg2":
+        altList.append(quick_figure("psycopg2-binary",pkgVer))
+    elif pkgName == "torch":
+        altList.append("-i https://download.pytorch.org/whl/cpu")
+        altList.append(pkg)
+
+
+def transcribe(envLoc: str, envName: str):
+    with open(envLoc + "environment-"+ envName + ".yml", "r") as baseEnvFp:
+            normalDeps=[]
+            devDeps=[]
+
+            condaDeps : list = yaml.load(baseEnvFp, yaml.Loader)["dependencies"]
+            for dep in condaDeps[:-1]:
+                dep = dep.replace("=", "==")
+
+                normalDeps.append(dep)
+                dev_alt(devDeps, dep)
+
+
+            if type(condaDeps[-1]) is dict and "pip" in condaDeps[-1].keys():
+                for dep in condaDeps[-1]["pip"]:
+                    normalDeps.append(dep)
+                    dev_alt(devDeps, dep)
+
+            #FIXME: Sometimes this doesn't update existing req files
+            with open(envLoc+"reqs.txt", "w") as computedReqs:
+                for line in normalDeps:
+                    computedReqs.write(line + "\n")
+
+            print(devDeps)
+            with open(envLoc+"reqsDev.txt", "w") as computedDevReqs:
+                for line in devDeps:
+                    computedDevReqs.write(line + "\n")
+
+def main() -> None:
+    envList = [[a+"/",a] for a in ["attacker", "honeypot", "manager"]]
+    envList.append(["","core"])
+
+    for r in envList:
+        transcribe(*r)
+
+
+
+if __name__ == "__main__":
+    main()