diff --git a/compose/local/__init__.py b/compose/local/__init__.py index bd93e7bd..4e86eacf 100644 --- a/compose/local/__init__.py +++ b/compose/local/__init__.py @@ -45,7 +45,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 7 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 1 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/compose/local/dask/Dockerfile b/compose/local/dask/Dockerfile index 102b2e51..0c4078aa 100644 --- a/compose/local/dask/Dockerfile +++ b/compose/local/dask/Dockerfile @@ -1,4 +1,4 @@ -FROM daskdev/dask:2024.5.1-py3.12 +FROM daskdev/dask:2024.5.2-py3.12 ENV DEBIAN_FRONTEND noninteractive ARG local_folder=/uploads @@ -47,7 +47,7 @@ RUN python setup.py build \ # Workers should have similar reqs as django WORKDIR / COPY ./requirements /requirements -RUN pip install uv==0.1.44 -e git+https://github.com/volatilityfoundation/volatility3.git@dc7a3878fa39156d89d567c3e823f1956675f192#egg=volatility3 \ +RUN pip install uv==0.2.10 -e git+https://github.com/volatilityfoundation/volatility3.git@543a39485bdf57df47d731b55ab112e04f3033f0#egg=volatility3 \ && uv pip install --no-cache --system -r /requirements/base.txt COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 7aa05de3..b21fdd95 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -65,7 +65,7 @@ RUN /usr/local/go/bin/go build FROM common-base WORKDIR / COPY ./requirements /requirements -RUN pip install uv==0.1.44 -e git+https://github.com/volatilityfoundation/volatility3.git@dc7a3878fa39156d89d567c3e823f1956675f192#egg=volatility3 \ +RUN pip install uv==0.2.10 -e git+https://github.com/volatilityfoundation/volatility3.git@543a39485bdf57df47d731b55ab112e04f3033f0#egg=volatility3 \ && uv pip install --no-cache --system -r /requirements/base.txt COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py diff --git a/orochi/api/api.py b/orochi/api/api.py index f008272a..2cafa4af 100644 --- a/orochi/api/api.py +++ b/orochi/api/api.py @@ -5,6 +5,7 @@ from orochi.api.routers.dumps import router as dumps_router from orochi.api.routers.folders import router as folders_router from orochi.api.routers.plugins import router as plugins_router +from orochi.api.routers.rules import router as rules_router from orochi.api.routers.users import router as users_router from orochi.api.routers.utils import router as utils_router @@ -16,3 +17,4 @@ api.add_router("/plugins/", plugins_router, tags=["Plugins"]) api.add_router("/utils/", utils_router, tags=["Utils"]) api.add_router("/bookmarks/", bookmarks_router, tags=["Bookmarks"]) +api.add_router("/rules/", rules_router, tags=["Rules"]) diff --git a/orochi/api/filters.py b/orochi/api/filters.py index 9dbadb37..f19878c9 100644 --- a/orochi/api/filters.py +++ b/orochi/api/filters.py @@ -1,6 +1,8 @@ from enum import Enum +from typing import List from ninja import Schema +from pydantic import root_validator class OPERATING_SYSTEM(str, Enum): @@ -16,3 +18,36 @@ class OperatingSytemFilters(Schema): class DumpFilters(Schema): result: int = None + + +################################################### +# Rules +################################################### +class Search(Schema): + value: str = None + regex: bool = False + + +class Column(Schema): + data: int + name: str = None + searchable: bool = True + orderable: bool = True + search: Search = None + + +class Order(Schema): + column: int = 0 + dir: str = "asc" + + +class RulesFilter(Schema): + start: int = 0 + length: int = 10 + columns: List[Column] = [] + search: Search = None + order: List[Order] = [] + + @root_validator(pre=True) + def extract_data(cls, v): + return v diff --git a/orochi/api/models.py b/orochi/api/models.py index a87d2fed..9fa310e5 100644 --- a/orochi/api/models.py +++ b/orochi/api/models.py @@ -7,6 +7,7 @@ from orochi.website.defaults import OSEnum from orochi.website.models import Bookmark, Dump, Folder, Plugin +from orochi.ya.models import Rule ################################################### # Auth @@ -233,3 +234,21 @@ class BookmarksInSchema(Schema): icon: str = None selected_plugin: str = None query: Optional[str] = None + + +################################################### +# Rules +################################################### +class RuleBuildSchema(Schema): + rule_ids: List[int] + rulename: str + + +class RulesOutSchema(ModelSchema): + class Meta: + model = Rule + fields = ["id", "path", "enabled", "compiled", "ruleset", "created", "updated"] + + +class ListStr(Schema): + rule_ids: List[int] diff --git a/orochi/api/routers/rules.py b/orochi/api/routers/rules.py new file mode 100644 index 00000000..c979e9e7 --- /dev/null +++ b/orochi/api/routers/rules.py @@ -0,0 +1,153 @@ +import os +from typing import List + +import yara +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from ninja import Query, Router +from ninja.security import django_auth + +from orochi.api.filters import RulesFilter +from orochi.api.models import ( + ErrorsOut, + ListStr, + RuleBuildSchema, + RulesOutSchema, + SuccessResponse, +) +from orochi.website.models import CustomRule +from orochi.ya.models import Rule + +router = Router() + + +@router.get("/", response={200: List[RulesOutSchema]}, auth=django_auth) +def list_rules(request, filters: Query[RulesFilter]): + return Rule.objects.all() + + +@router.get("/{pk}/download", auth=django_auth) +def download(request, pk: int): + """ + Download a rule file by its primary key. + + Args: + pk (int): The primary key of the rule to download. + + Returns: + HttpResponse: The HTTP response object containing the downloaded rule file. + + Raises: + Exception: If an error occurs during the process. + """ + try: + rule = Rule.objects.filter(pk=pk).filter(ruleset__enabled=True) + if rule.count() == 1: + rule = rule.first() + else: + return 400, {"errors": "Generic error"} + if os.path.exists(rule.path): + with open(rule.path, "rb") as f: + rule_data = f.read() + + response = HttpResponse( + rule_data, + content_type="application/text", + ) + response["Content-Disposition"] = ( + f"attachment; filename={os.path.basename(rule.path)}" + ) + return response + else: + return 400, {"errors": "Rule not found"} + except Exception as excp: + return 400, {"errors": str(excp)} + + +@router.delete( + "/", + auth=django_auth, + url_name="delete_rules", + response={200: SuccessResponse, 400: ErrorsOut}, +) +def delete_rules(request, info: ListStr): + """ + Summary: + Delete rules based on the provided rule IDs. + + Explanation: + This function deletes rules based on the specified rule IDs belonging to the authenticated user. It removes the rules from the database and returns a success message upon deletion. + + Args: + - request: The request object. + - rule_ids: A list of integers representing the IDs of rules to be deleted. + + Returns: + - Tuple containing status code and a message dictionary. + + Raises: + - Any exception encountered during the process will result in a 400 status code with an error message. + """ + try: + rules = Rule.objects.filter(pk__in=info.rule_ids, ruleset__user=request.user) + rules.delete() + rules_count = rules.count() + if rules_count == 0: + return 200, {"message": f"{rules_count} rules deleted."} + else: + return 200, {"message": "Only rules in your ruleset can be deleted."} + except Exception as excp: + return 400, { + "errors": str(excp) if excp else "Generic error during rules deletion" + } + + +@router.post( + "/build", + response={200: SuccessResponse, 400: ErrorsOut}, + url_name="rule_build", + auth=django_auth, +) +def build_rules(request, info: RuleBuildSchema): + """ + Summary: + Build rules based on the provided information. + + Explanation: + This function builds rules using the provided information and saves them in a custom folder. It creates a new YARA rule file and stores it in the specified location. + + Args: + - request: The request object. + - info: An instance of RuleBuildSchema containing rule information. + + Returns: + - Tuple containing status code and a message dictionary. + + Raises: + - Any exception encountered during the process will result in a 400 status code with an error message. + """ + try: + rules = Rule.objects.filter(pk__in=info.rule_ids) + rules_file = {f"{rule.ruleset.name}_{rule.pk}": rule.path for rule in rules} + rules = yara.compile(filepaths=rules_file) + + # Manage duplicated file path + folder = f"/yara/customs/{request.user.username}" + os.makedirs(folder, exist_ok=True) + new_path = f"{folder}/{info.rulename}.yara" + filename, extension = os.path.splitext(new_path) + counter = 1 + while os.path.exists(new_path): + new_path = f"{filename}{counter}{extension}" + counter += 1 + + rules.save(new_path) + CustomRule.objects.create( + user=request.user, + path=new_path, + name=info.rulename, + ) + + return 200, {"message": f"Rule {info.rulename} created"} + except Exception as excp: + return 400, {"errors": str(excp)} diff --git a/orochi/templates/users/user_bookmarks.html b/orochi/templates/users/user_bookmarks.html index eda778fa..50088099 100644 --- a/orochi/templates/users/user_bookmarks.html +++ b/orochi/templates/users/user_bookmarks.html @@ -174,7 +174,7 @@ delay: 5000 }); }, - error: function () { + error: function (data) { $.toast({ title: 'Bookmark status!', content: data.message, @@ -217,7 +217,7 @@ delay: 5000 }); }, - error: function () { + error: function (data) { $.toast({ title: 'Bookmark status!', content: data.message, diff --git a/orochi/templates/users/user_rules.html b/orochi/templates/users/user_rules.html index a0cdb6d7..db724baa 100644 --- a/orochi/templates/users/user_rules.html +++ b/orochi/templates/users/user_rules.html @@ -143,7 +143,7 @@ { sortable: false, render: function (data, type, row, meta) { - let down = ``; + let down = ``; down += ``; return down; } @@ -204,15 +204,36 @@ if (rows_selected.length > 0) { bootbox.confirm("Delete selected rules in your ruleset?", function (result) { table.column(0).checkboxes.deselectAll(); + let obj = {}; var items = []; rows_selected.each(function (val) { items.push(val) }); + obj['rule_ids'] = items; + + $.ajaxSetup({ + headers: { 'X-CSRFToken': $('input[name="csrfmiddlewaretoken"]').val() } + }); + $.ajax({ - url: "{% url 'ya:delete' %}", - method: 'get', - data: { 'rules': items }, + url: "{% url 'api:delete_rules' %}", + method: 'delete', + data: JSON.stringify(obj), dataType: 'json', success: function (data) { table.ajax.reload(); + $.toast({ + title: 'Rules Deleted!', + content: data.message, + type: 'success', + delay: 5000 + }); + }, + error: function (data) { + $.toast({ + title: 'Delete Rules error!', + content: data.message, + type: 'error', + delay: 5000 + }); } }); }); @@ -226,11 +247,17 @@ bootbox.prompt("Select name for compiled custom rule:", function (result) { var items = []; rows_selected.each(function (val) { items.push(val) }); - var items_str = items.join(';'); + let obj = { + 'rule_ids': items, + 'rulename': result + }; + $.ajaxSetup({ + headers: { 'X-CSRFToken': $('input[name="csrfmiddlewaretoken"]').val() } + }); $.ajax({ - url: "{% url 'ya:build' %}", + url: "{% url 'api:rule_build' %}", method: 'post', - data: { 'rules': items_str, 'rulename': result, 'csrfmiddlewaretoken': $("input[name=csrfmiddlewaretoken").val() }, + data: JSON.stringify(obj), dataType: 'json', beforeSend: function () { if (items.length > 50) { @@ -246,6 +273,21 @@ if (items.length > 50) { dialog.modal('hide'); } + $.toast({ + title: 'Build Rule status!', + content: data.message, + type: 'success', + delay: 5000 + }); + }, + error: function (data) { + $.toast({ + title: 'Build Rule Error!', + content: data.message, + type: 'error', + delay: 5000 + }); + $("#modal-update").modal('hide'); } }); }); diff --git a/orochi/ya/models.py b/orochi/ya/models.py index c05057e1..dba65e27 100644 --- a/orochi/ya/models.py +++ b/orochi/ya/models.py @@ -1,7 +1,8 @@ -from django.db import models from django.contrib.auth import get_user_model -from django.db.models.signals import post_save, post_delete +from django.db import models +from django.db.models.signals import post_delete, post_save from django.dispatch import receiver + from orochi.ya.schema import RuleIndex @@ -34,7 +35,7 @@ class Rule(models.Model): ruleset = models.ForeignKey(Ruleset, on_delete=models.CASCADE, related_name="rules") def __str__(self): - return "[{}] {}".format(self.ruleset.name, self.path) + return f"[{self.ruleset.name}] {self.path}" @receiver(post_save, sender=Rule) diff --git a/orochi/ya/schema.py b/orochi/ya/schema.py index 171492b4..135f24cf 100644 --- a/orochi/ya/schema.py +++ b/orochi/ya/schema.py @@ -1,3 +1,4 @@ +import contextlib from pathlib import Path import elasticsearch @@ -34,10 +35,8 @@ def __init__(self): def create_index(self): if not self.es_client.indices.exists(index=self.index_name): - try: + with contextlib.suppress(elasticsearch.exceptions.RequestError): self.es_client.indices.create(index=self.index_name, body=self.schema) - except elasticsearch.exceptions.RequestError: - pass def delete_index(self): if self.es_client.indices.exists(index=self.index_name): diff --git a/orochi/ya/urls.py b/orochi/ya/urls.py index 26d7b9a8..348037e0 100644 --- a/orochi/ya/urls.py +++ b/orochi/ya/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from orochi.ya import views app_name = "ya" @@ -11,8 +12,6 @@ ), path("list", views.list_rules, name="list"), path("upload", views.upload, name="upload"), - path("delete", views.delete, name="delete"), - path("build", views.build, name="build"), path("detail", views.detail, name="detail"), path("download_rule/", views.download_rule, name="download_rule"), ] diff --git a/orochi/ya/views.py b/orochi/ya/views.py index 6b61c3d1..a4086b50 100644 --- a/orochi/ya/views.py +++ b/orochi/ya/views.py @@ -2,7 +2,6 @@ import shutil from pathlib import Path -import yara from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -12,8 +11,8 @@ from django.http.response import HttpResponse from django.shortcuts import get_object_or_404, redirect from django.template.loader import render_to_string +from django.views.decorators.http import require_http_methods -from orochi.website.models import CustomRule from orochi.ya.forms import EditRuleForm, RuleForm from orochi.ya.models import Rule, Ruleset from orochi.ya.schema import RuleIndex @@ -96,86 +95,13 @@ def list_rules(request): return JsonResponse(return_data) -@login_required -def build(request): - """ - Creates fat yara from selected rules - """ - rules_id = request.POST.get("rules").split(";") - rulename = request.POST.get("rulename") - - rules = Rule.objects.filter(pk__in=rules_id) - - rules_file = {f"{rule.ruleset.name}_{rule.pk}": rule.path for rule in rules} - - rules = yara.compile(filepaths=rules_file) - - # Manage duplicated file path - folder = f"/yara/customs/{request.user.username}" - os.makedirs(folder, exist_ok=True) - new_path = f"{folder}/{rulename}.yara" - filename, extension = os.path.splitext(new_path) - counter = 1 - while os.path.exists(new_path): - new_path = f"{filename}{counter}{extension}" - counter += 1 - - rules.save(new_path) - CustomRule.objects.create( - user=request.user, - path=new_path, - name=rulename, - ) - - return JsonResponse({"ok": True}) - - -@login_required -def delete(request): - """ - Delete selected rules if in your ruleset - """ - rules_id = request.GET.getlist("rules[]") - rules = Rule.objects.filter(pk__in=rules_id, ruleset__user=request.user) - rules.delete() - return JsonResponse({"ok": True}) - - +@require_http_methods(["GET"]) @login_required def detail(request): """ Return content of rule """ data = {} - if request.method == "POST": - form = EditRuleForm(data=request.POST) - if form.is_valid(): - pk = request.POST.get("pk") - rule = get_object_or_404(Rule, pk=pk) - if rule.ruleset.user == request.user: - with open(rule.path, "w") as f: - f.write(request.POST.get("text")) - else: - ruleset = get_object_or_404(Ruleset, user=request.user) - user_path = ( - f"{settings.LOCAL_YARA_PATH}/{request.user.username}-Ruleset" - ) - os.makedirs(user_path, exist_ok=True) - rule.pk = None - rule.ruleset = ruleset - new_path = f"{user_path}/{Path(rule.path).name}" - filename, extension = os.path.splitext(new_path) - counter = 1 - while os.path.exists(new_path): - new_path = f"{filename}{counter}{extension}" - counter += 1 - with open(new_path, "w") as f: - f.write(request.POST.get("text")) - rule.path = new_path - rule.save() - return JsonResponse({"ok": True}) - raise Http404 - pk = request.GET.get("pk") rule = get_object_or_404(Rule, pk=pk) try: @@ -242,26 +168,3 @@ def upload(request): request=request, ) return JsonResponse(data) - - -@login_required -def download_rule(request, pk): - """ - Download selected rule - """ - rule = Rule.objects.filter(pk=pk).filter(ruleset__enabled=True) - if rule.count() == 1: - rule = rule.first() - else: - raise Http404 - - if os.path.exists(rule.path): - with open(rule.path, "rb") as fh: - response = HttpResponse( - fh.read(), content_type="application/force-download" - ) - response[ - "Content-Disposition" - ] = f"inline; filename={os.path.basename(rule.path)}" - return response - raise Http404("404") diff --git a/requirements/base.txt b/requirements/base.txt index 1cd7be5a..e5f24798 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,7 +7,7 @@ argon2-cffi==23.1.0 # https://github.com/evansd/whitenoise whitenoise==6.6.0 # https://github.com/andymccurdy/redis-py -redis==5.0.4 +redis==5.0.5 # https://github.com/redis/hiredis-py hiredis==2.3.2 # https://github.com/psycopg/psycopg2 @@ -28,7 +28,7 @@ channels_redis==4.2.0 # https://github.com/joke2k/django-environ django-environ==0.11.2 # https://github.com/pennersr/django-allauth -django-allauth==0.63.1 +django-allauth==0.63.3 # https://github.com/django-crispy-forms/django-crispy-forms django-crispy-forms==2.1 # https://github.com/jazzband/django-redis @@ -64,18 +64,18 @@ django-admin-multiple-choice-list-filter==0.1.1 # Elasticsearch # ------------------------------------------------------------------------------ # https://github.com/elastic/elasticsearch-py -elasticsearch==8.13.1 +elasticsearch==8.14.0 # https://github.com/elastic/elasticsearch-dsl-py -elasticsearch-dsl==8.13.1 +elasticsearch-dsl==8.14.0 # https://github.com/jurismarches/luqum luqum==0.13.0 # Dask & co # ------------------------------------------------------------------------------ # https://github.com/dask/dask -dask==2024.5.1 +dask==2024.5.2 # https://github.com/dask/distributed -distributed==2024.5.1 +distributed==2024.5.2 # https://msgpack.org/ TO BE ALIGNED WITH SCHEDULER msgpack==1.0.8 # https://github.com/python-lz4/python-lz4 @@ -87,7 +87,7 @@ cloudpickle==3.0.0 # https://pypi.org/project/toolz/ toolz==0.12.1 # https://pypi.org/project/tornado/ -tornado==6.4 +tornado==6.4.1 # https://pandas.pydata.org/ pandas==2.2.2 @@ -102,7 +102,8 @@ clamdpy==0.1.0.post1 # https://github.com/VirusTotal/vt-py vt-py==0.18.2 # https://github.com/mkorman90/regipy/ -regipy[full]==4.2.1 +#regipy[full]==4.2.1 +regipy[full] @ git+https://github.com/dadokkio/regipy@6d11c7dc4ef864d13bc84bc23ef2ca80cb0a90f0 # http://www.capstone-engine.org/ capstone==5.0.1 # https://github.com/Julian/jsonschema @@ -133,7 +134,7 @@ pefile==2023.2.7 # misp export # ------------------------------------------------------------------------------ # https://github.com/MISP/PyMISP -pymisp==2.4.190 +pymisp==2.4.193 # ldap # ------------------------------------------------------------------------------ diff --git a/requirements/local.txt b/requirements/local.txt index e3a5c767..78b8a27e 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -3,9 +3,9 @@ # Testing # ------------------------------------------------------------------------------ # https://github.com/python/mypy -mypy==1.9.0 +mypy==1.10.0 # https://github.com/typeddjango/django-stubs -django-stubs==5.0.0 +django-stubs==5.0.2 # https://github.com/pytest-dev/pytest pytest==8.1.1 # https://github.com/Frozenball/pytest-sugar @@ -18,7 +18,7 @@ flake8==7.0.0 # https://github.com/gforcada/flake8-isort flake8-isort==6.1.1 # https://github.com/nedbat/coveragepy -coverage==7.5.1 +coverage==7.5.3 # https://github.com/ambv/black black==24.4.2 # https://github.com/PyCQA/pylint-django @@ -31,7 +31,7 @@ pre-commit==3.7.1 # https://github.com/FactoryBoy/factory_boy factory-boy==3.3.0 # https://github.com/jazzband/django-debug-toolbar -django-debug-toolbar==4.3.0 +django-debug-toolbar==4.4.2 # https://github.com/django-extensions/django-extensions django-extensions==3.2.3 # https://github.com/nedbat/django_coverage_plugin