From 4aabe0645850ee0136d41c1a27fda58239153cf0 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 21 Mar 2024 15:37:24 +0100 Subject: [PATCH] improve plugin install, regipy plugins --- orochi/templates/base.html | 12 +-- orochi/templates/website/index.html | 1 + orochi/templates/website/partial_folder.html | 2 +- orochi/templates/website/partial_info.html | 80 ++++++++++++++----- orochi/utils/plugin_install.py | 6 +- orochi/utils/volatility_dask_elk.py | 68 ++++++++++++---- orochi/website/forms.py | 34 +++++--- .../migrations/0054_dump_regipy_plugins.py | 25 ++++++ orochi/website/models.py | 3 + orochi/website/views.py | 6 +- requirements/base.txt | 2 +- 11 files changed, 184 insertions(+), 55 deletions(-) create mode 100644 orochi/website/migrations/0054_dump_regipy_plugins.py diff --git a/orochi/templates/base.html b/orochi/templates/base.html index ccba01be..70c59b71 100644 --- a/orochi/templates/base.html +++ b/orochi/templates/base.html @@ -204,12 +204,14 @@ // TASK STATUS setInterval(function () { - $.get("{% url 'website:dask_status' %}").done( - function (data) { + $.ajax({ + url: "{% url 'website:dask_status' %}", + success: function (data) { $("#tasks_running").html(data.running); - } - ) - }, 5000); + }, + timeout: 2000 + }); + }, 10000); // ABOUT $(document).on("click", "#about", function () { diff --git a/orochi/templates/website/index.html b/orochi/templates/website/index.html index 29beec51..1ea55af1 100644 --- a/orochi/templates/website/index.html +++ b/orochi/templates/website/index.html @@ -716,6 +716,7 @@
History Log
}); update_sidebar(); update_main_stage(); + selected_plugin = null; $.toast({ title: 'Index delete!', content: 'Index has been deleted successfully.', diff --git a/orochi/templates/website/partial_folder.html b/orochi/templates/website/partial_folder.html index d5bccc09..272ea6ec 100644 --- a/orochi/templates/website/partial_folder.html +++ b/orochi/templates/website/partial_folder.html @@ -4,7 +4,7 @@ {{ form.media }} {% csrf_token %}
-
Create a new foledr
+
Create a new folder
diff --git a/orochi/templates/website/partial_info.html b/orochi/templates/website/partial_info.html index 923e98bf..9dc97c81 100644 --- a/orochi/templates/website/partial_info.html +++ b/orochi/templates/website/partial_info.html @@ -1,25 +1,67 @@
-

{{dump.name}}

+

+ {{dump.name}} +

-
-
md5
-
{{dump.md5}}
-
sha256
-
{{dump.sha256}}
-
Size
-
{{dump.size}}
-
Index
-
{{dump.index}}
-
Filepath
-
{{dump.upload}}
- {% if dump.comment %}
Comment
-
{{dump.comment}}
{% endif %} - {% if dump.banner %}
Banner
-
{{dump.banner}}
{% endif %} - {% if dump.suggested_symbols_path %}
Suggested Symbol Path
-
{{dump.suggested_symbols_path.0}}
{% endif %} -
+ + {% if dump.regipy_plugins %} + + {% endif %} +
+
+
+
md5
+
{{dump.md5}}
+
sha256
+
{{dump.sha256}}
+
Size
+
{{dump.size}}
+
Index
+
{{dump.index}}
+
Filepath
+
{{dump.upload}}
+ {% if dump.comment %}
Comment
+
{{dump.comment}}
{% endif %} + {% if dump.banner %}
Banner
+
{{dump.banner}}
{% endif %} + {% if dump.suggested_symbols_path %}
Suggested Symbol Path
+
{{dump.suggested_symbols_path.0}}
{% endif %} +
+
+
+ {% if dump.regipy_plugins %} +
+ {% for item in dump.regipy_plugins %} +
+

+ +

+
+
+ 
{{item.data|pprint}}
+
+
+
+ {% endfor %} +
+ {% endif %} +
+
diff --git a/orochi/utils/plugin_install.py b/orochi/utils/plugin_install.py index 04db9994..ebced56e 100644 --- a/orochi/utils/plugin_install.py +++ b/orochi/utils/plugin_install.py @@ -54,9 +54,11 @@ def install_process(bash_script, reqs_script, tmp_folder): _ = contexts.Context() _ = framework.import_files(volatility3.plugins, True) plugin_names = [ - x for x in framework.list_plugins() if x.startswith(f"custom.{py_name}") + {x: y} + for x, y in framework.list_plugins().items() + if x.startswith(f"custom.{py_name}") ] - logging.debug(f"Installed plugins: {','.join(plugin_names)}") + logging.debug("Plugins installed successfully.") return plugin_names except Exception as e: logging.error(f"Error installing plugin: {e}") diff --git a/orochi/utils/volatility_dask_elk.py b/orochi/utils/volatility_dask_elk.py index 4f010840..1705ad2f 100644 --- a/orochi/utils/volatility_dask_elk.py +++ b/orochi/utils/volatility_dask_elk.py @@ -28,7 +28,12 @@ from django.conf import settings from elasticsearch import Elasticsearch, helpers from elasticsearch_dsl import Search -from regipy.exceptions import RegistryParsingException +from regipy.exceptions import ( + NoRegistrySubkeysException, + RegistryKeyNotFoundException, + RegistryParsingException, +) +from regipy.plugins.utils import dump_hive_to_json, run_relevant_plugins from regipy.registry import RegistryHive from volatility3 import cli, framework from volatility3.cli.text_renderer import ( @@ -252,7 +257,7 @@ async def run_vt(filepath): to_check = hash_checksum(filepath)[0] report = await client.get_object_async(f"/files/{to_check}") if report := report.to_dict().get("attributes"): - stats = {k: v for k, v in report.get("last_analysis_stats", {}).items()} + stats = dict(report.get("last_analysis_stats", {}).items()) if scan_date := report.get("last_analysis_date"): scan_date = datetime.datetime.fromtimestamp(scan_date).strftime( "%m/%d/%Y" @@ -272,12 +277,14 @@ async def run_vt(filepath): return -def run_regipy(filepath): +def run_regipy(filepath, plugins=False): """ Runs regipy on filepath """ try: registry_hive = RegistryHive(filepath) + *a, index, _, hive_name = filepath.split("/") + dump = Dump.objects.get(index=index) data = [] try: data.extend( @@ -288,15 +295,32 @@ def run_regipy(filepath): ) except RegistryParsingException as e: logging.error(e) - root = {"values": data} - root = json.loads(json.dumps(root).replace(r"\u0000", "")) with open(f"{filepath}.regipy.json", "w") as f: - json.dump(root, f) + json.dump(json.loads(json.dumps(data).replace(r"\u0000", "")), f) + if plugins: + try: + if plugins_data := run_relevant_plugins(registry_hive, as_json=True): + for plugin, plugin_data in plugins_data.items(): + if plugin_data: + info = { + "hive": hive_name, + "plugin": plugin, + "data": plugin_data, + } + dump.regipy_plugins.append(info) + dump.save() + except ( + ModuleNotFoundError, + RegistryParsingException, + RegistryKeyNotFoundException, + NoRegistrySubkeysException, + ) as e: + logging.error(e) except Exception as e: logging.error(e) -def run_plugin(dump_obj, plugin_obj, params=None, user_pk=None): +def run_plugin(dump_obj, plugin_obj, params=None, user_pk=None, regipy_plugins=False): """ Execute a single plugin on a dump with optional params. If success data are sent to elastic. @@ -324,8 +348,8 @@ def run_plugin(dump_obj, plugin_obj, params=None, user_pk=None): ctx.config["automagic.LayerStacker.stackers"] = stacker.choose_os_stackers( plugin ) - # LOCAL DUMPS REQUIRES FILES - local_dump = plugin_obj.local_dump + # LOCAL DUMPS REQUIRES FILES - Also regipy plugins + local_dump = plugin_obj.local_dump or regipy_plugins # ADD PARAMETERS, AND IF LOCAL DUMP ENABLE ADD DUMP TRUE BY DEFAULT plugin_config_path = interfaces.configuration.path_join( @@ -508,14 +532,16 @@ def run_plugin(dump_obj, plugin_obj, params=None, user_pk=None): ) # RUN VT AND REGIPY ON CREATED FILES - if plugin_obj.vt_check or plugin_obj.regipy_check: + if plugin_obj.vt_check or plugin_obj.regipy_check or regipy_plugins: dask_client = get_client() for file_id in file_list: output_path = f"{local_path}/{file_id.preferred_filename}" if plugin_obj.vt_check: fire_and_forget(dask_client.submit(run_vt, output_path)) - if plugin_obj.regipy_check: - fire_and_forget(dask_client.submit(run_regipy, output_path)) + if plugin_obj.regipy_check or regipy_plugins: + fire_and_forget( + dask_client.submit(run_regipy, output_path, regipy_plugins) + ) # EVERYTHING OK result = Result.objects.get(plugin=plugin_obj, dump=dump_obj) @@ -790,6 +816,7 @@ def unzip_then_run(dump_pk, user_pk, password, restart, move): if banner: banner.result = RESULT_STATUS_RUNNING banner.save() + logging.info(f"[dump {dump_pk}] Running banners plugin") run_plugin(dump, banner.plugin) time.sleep(1) banner_result = get_banner(banner) @@ -799,16 +826,25 @@ def unzip_then_run(dump_pk, user_pk, password, restart, move): f"[dump {dump_pk}] guessed banner '{dump.banner}'" ) dump.save() + elif dump.operating_system == "Windows": + regipy = dump.result_set.get( + plugin__name="windows.registry.hivelist.HiveList" + ) + logging.info(f"[dump {dump_pk}] Running regipy plugins") + run_plugin(dump, regipy.plugin, regipy_plugins=True) if restart or check_runnable(dump.pk, dump.operating_system, dump.banner): dask_client = get_client() secede() tasks = [] - tasks_list = ( + if dump.operating_system == "Linux": + tasks_list = dump.result_set.exclude(plugin__name="banners.Banners") + elif dump.operating_system == "Windows": + tasks_list = dump.result_set.exclude( + plugin__name="windows.registry.hivelist.HiveList" + ) + else: dump.result_set.all() - if dump.operating_system != "Linux" - else dump.result_set.exclude(plugin__name="banners.Banners") - ) if restart: tasks_list = tasks_list.filter(plugin__pk__in=restart) for result in tasks_list: diff --git a/orochi/website/forms.py b/orochi/website/forms.py index bec06437..8108e7b2 100644 --- a/orochi/website/forms.py +++ b/orochi/website/forms.py @@ -12,7 +12,8 @@ ) from orochi.utils.plugin_install import plugin_install -from orochi.website.models import Bookmark, Dump, Folder, Plugin +from orochi.website.defaults import RESULT_STATUS_NOT_STARTED +from orochi.website.models import Bookmark, Dump, Folder, Plugin, Result, UserPlugin ###################################### @@ -162,7 +163,7 @@ def __init__(self, *args, **kwargs): required=not field["optional"], ) self.fields[field["name"]].help_text = ( - "List of '{}' comma separated".format(field["type"].__name__) + f"""List of '{field["type"].__name__}' comma separated""" ) @@ -223,13 +224,28 @@ class Meta: def save(self, commit=True): plugin = self.cleaned_data["plugin"] - plugin_name = plugin_install(plugin.file.path) - plugin = super(PluginCreateAdminForm, self).save(commit=commit) - plugin.name = plugin_name - plugin.local = True - plugin.local_date = datetime.now() - plugin.save() - return plugin + plugin_names = plugin_install(plugin.file.path) + first = None + for plugin_data in plugin_names: + plugin_name, plugin_class = list(plugin_data.items())[0] + plugin = super(PluginCreateAdminForm, self).save(commit=commit) + plugin.comment = plugin_class.__doc__ + plugin.name = plugin_name + plugin.local = True + plugin.local_date = datetime.now() + plugin.save() + for user in get_user_model().objects.all(): + UserPlugin.objects.get_or_create(user=user, plugin=plugin) + for dump in Dump.objects.all(): + if plugin.operating_system in [dump.operating_system, "Other"]: + Result.objects.update_or_create( + dump=dump, + plugin=plugin, + defaults={"result": RESULT_STATUS_NOT_STARTED}, + ) + if not first: + first = plugin + return first class PluginEditAdminForm(FileFormMixin, forms.ModelForm): diff --git a/orochi/website/migrations/0054_dump_regipy_plugins.py b/orochi/website/migrations/0054_dump_regipy_plugins.py new file mode 100644 index 00000000..ad804ccc --- /dev/null +++ b/orochi/website/migrations/0054_dump_regipy_plugins.py @@ -0,0 +1,25 @@ +# Generated by Django 5.0.3 on 2024-03-20 13:59 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("website", "0053_create_default_groups"), + ] + + operations = [ + migrations.AddField( + model_name="dump", + name="regipy_plugins", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.JSONField(blank=True, null=True), + blank=True, + null=True, + size=None, + default=list, + ), + ), + ] diff --git a/orochi/website/models.py b/orochi/website/models.py index 5fae3808..c81c53dd 100644 --- a/orochi/website/models.py +++ b/orochi/website/models.py @@ -89,6 +89,9 @@ class Dump(models.Model): ) banner = models.CharField(max_length=500, blank=True, null=True) upload = models.FileField(upload_to="uploads") + regipy_plugins = ArrayField( + models.JSONField(blank=True, null=True), blank=True, null=True, default=list + ) folder = models.ForeignKey(Folder, on_delete=models.SET_NULL, blank=True, null=True) comment = models.TextField(blank=True, null=True) description = models.TextField(blank=True, null=True) diff --git a/orochi/website/views.py b/orochi/website/views.py index 216a89c1..01d67cfc 100644 --- a/orochi/website/views.py +++ b/orochi/website/views.py @@ -327,10 +327,12 @@ def install_plugin(request): f.write(r.content) f.close() if plugin_names := plugin_install(f.name): - for plugin_name in plugin_names: + for plugin_data in plugin_names: + plugin_name, plugin_class = list(plugin_data.items())[0] plugin, _ = Plugin.objects.update_or_create( name=plugin_name, defaults={ + "comment": plugin_class.__doc__, "operating_system": operating_system, "local": True, "local_date": datetime.now(), @@ -839,7 +841,7 @@ def json_view(request, filepath): ): raise Http404("404") with open(filepath, "r") as f: - values = json.load(f).get("values", []) + values = json.load(f) context = {"data": json.dumps(values)} return render(request, "website/json_view.html", context) diff --git a/requirements/base.txt b/requirements/base.txt index 5903afb4..2af96431 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -106,7 +106,7 @@ clamdpy==0.1.0.post1 # https://github.com/VirusTotal/vt-py vt-py==0.18.0 # https://github.com/mkorman90/regipy/ -regipy==4.1.1 +regipy[full]==4.1.1 # http://www.capstone-engine.org/ capstone==5.0.1 # https://github.com/Julian/jsonschema