From 42a0c922b3987d0bca8e79c2650d0a95d35e3c4a Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 24 Mar 2026 01:21:20 +0530 Subject: [PATCH 1/5] ecurely store plugin secrets encrypted at rest --- api_app/classes.py | 52 +++-- .../0073_encrypt_plugin_config_secrets.py | 65 ++++++ api_app/models.py | 220 ++++++++++++++---- intel_owl/settings/security.py | 10 + requirements/project-requirements.txt | 1 + .../api_app/test_plugin_config_encryption.py | 103 ++++++++ 6 files changed, 384 insertions(+), 67 deletions(-) create mode 100644 api_app/migrations/0073_encrypt_plugin_config_secrets.py create mode 100644 tests/api_app/test_plugin_config_encryption.py diff --git a/api_app/classes.py b/api_app/classes.py index f347bcfd6d..0161f3cc69 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -7,6 +7,7 @@ import requests from billiard.exceptions import SoftTimeLimitExceeded +from certego_saas.apps.user.models import User from django.conf import settings from django.core.files import File from django.utils import timezone @@ -15,7 +16,6 @@ from api_app.decorators import abstractclassproperty, classproperty from api_app.models import AbstractReport, Job, PythonConfig, PythonModule -from certego_saas.apps.user.models import User logger = logging.getLogger(__name__) @@ -140,18 +140,28 @@ def __str__(self): return f"{self.__class__.__name__}" def config(self, runtime_configuration: typing.Dict): - """ - Configure the plugin with runtime parameters. - - Args: - runtime_configuration (dict): Runtime configuration parameters. - """ - self.__parameters = self._config.read_configured_params(self._user, runtime_configuration) + self.__parameters = self._config.read_configured_params( + self._user, runtime_configuration + ) for parameter in self.__parameters: - attribute_name = f"_{parameter.name}" if parameter.is_secret else parameter.name - setattr(self, attribute_name, parameter.value) + attribute_name = ( + f"_{parameter.name}" if parameter.is_secret else parameter.name + ) + value = parameter.value + # decrypt secrets that were stored encrypted + if ( + parameter.is_secret + and isinstance(value, str) + and value.startswith("gAAAAA") + ): + from api_app.models import PluginConfig + + value = PluginConfig._decrypt_value(value) + setattr(self, attribute_name, value) logger.debug( - f"Adding to {self.__class__.__name__} param {attribute_name} with value {parameter.value} " + f"Adding to {self.__class__.__name__} " + f"param {attribute_name} " + f"with value {parameter.value} " ) def before_run(self): @@ -206,7 +216,9 @@ def log_error(self, e): Args: e (Exception): The exception to log. """ - if isinstance(e, (*self.get_exceptions_to_catch(), SoftTimeLimitExceeded, HTTPError)): + if isinstance( + e, (*self.get_exceptions_to_catch(), SoftTimeLimitExceeded, HTTPError) + ): error_message = self.get_error_message(e) logger.error(error_message) else: @@ -225,7 +237,9 @@ def after_run_failed(self, e: Exception): self.report.status = self.report.STATUSES.FAILED self.report.save(update_fields=["status", "errors"]) if isinstance(e, HTTPError) and ( - hasattr(e, "response") and hasattr(e.response, "status_code") and e.response.status_code == 429 + hasattr(e, "response") + and hasattr(e.response, "status_code") + and e.response.status_code == 429 ): self.disable_for_rate_limit() else: @@ -265,7 +279,9 @@ def get_error_message(self, err, is_base_err=False): f" '{err}'" ) - def start(self, job_id: int, runtime_configuration: dict, task_id: str, *args, **kwargs): + def start( + self, job_id: int, runtime_configuration: dict, task_id: str, *args, **kwargs + ): """ Entrypoint function to execute the plugin. calls `before_run`, `run`, `after_run` @@ -403,7 +419,9 @@ def disable_for_rate_limit(self): self._user.membership.organization ) if org_configuration.rate_limit_timeout is not None: - api_key_parameter = self.__parameters.filter(name__contains="api_key").first() + api_key_parameter = self.__parameters.filter( + name__contains="api_key" + ).first() # if we do not have api keys OR the api key was org based # OR if the api key is not actually required and we do not have it set if ( @@ -413,7 +431,9 @@ def disable_for_rate_limit(self): ): org_configuration.disable_for_rate_limit() else: - logger.warning(f"Not disabling {self} because api key used is personal") + logger.warning( + f"Not disabling {self} because api key used is personal" + ) else: logger.warning( f"You are trying to disable {self} for rate limit without specifying a timeout." diff --git a/api_app/migrations/0073_encrypt_plugin_config_secrets.py b/api_app/migrations/0073_encrypt_plugin_config_secrets.py new file mode 100644 index 0000000000..74da405aae --- /dev/null +++ b/api_app/migrations/0073_encrypt_plugin_config_secrets.py @@ -0,0 +1,65 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import base64 +import hashlib +import json + +from django.conf import settings +from django.db import migrations + + +def _get_fernet(): + from cryptography.fernet import Fernet + + key = getattr(settings, "PLUGIN_CONFIG_FERNET_KEY", None) + if key is None: + key = base64.urlsafe_b64encode( + hashlib.sha256(settings.SECRET_KEY.encode()).digest() + ) + return Fernet(key) + + +def encrypt_existing_secrets(apps, schema_editor): + PluginConfig = apps.get_model("api_app", "PluginConfig") + fernet = _get_fernet() + + for pc in PluginConfig.objects.filter( + parameter__is_secret=True, + value__isnull=False, + ): + if isinstance(pc.value, str) and pc.value.startswith("gAAAAA"): + continue + encrypted = fernet.encrypt(json.dumps(pc.value).encode()).decode() + PluginConfig.objects.filter(pk=pc.pk).update(value=encrypted) + + +def decrypt_existing_secrets(apps, schema_editor): + PluginConfig = apps.get_model("api_app", "PluginConfig") + fernet = _get_fernet() + + for pc in PluginConfig.objects.filter( + parameter__is_secret=True, + value__isnull=False, + ): + if not (isinstance(pc.value, str) and pc.value.startswith("gAAAAA")): + continue + try: + decrypted = json.loads(fernet.decrypt(pc.value.encode()).decode()) + PluginConfig.objects.filter(pk=pc.pk).update(value=decrypted) + except Exception: + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("api_app", "0072_update_check_system"), + ] + + operations = [ + migrations.RunPython( + encrypt_existing_secrets, + reverse_code=decrypt_existing_secrets, + ), + ] diff --git a/api_app/models.py b/api_app/models.py index bd3dca557f..c5236901f6 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -48,6 +48,9 @@ if typing.TYPE_CHECKING: from api_app.classes import Plugin +from certego_saas.apps.organization.organization import Organization +from certego_saas.models import User + from api_app.decorators import classproperty from api_app.defaults import default_runtime from api_app.helpers import deprecated, get_now @@ -62,8 +65,6 @@ PythonConfigQuerySet, ) from api_app.validators import plugin_name_validator, validate_runtime_configuration -from certego_saas.apps.organization.organization import Organization -from certego_saas.models import User from intel_owl import tasks from intel_owl.celery import get_queue_name @@ -116,7 +117,9 @@ class PythonModule(models.Model): """ module = models.CharField(max_length=120, db_index=True) - base_path = models.CharField(max_length=120, db_index=True, choices=PythonModuleBasePaths.choices) + base_path = models.CharField( + max_length=120, db_index=True, choices=PythonModuleBasePaths.choices + ) update_schedule = models.ForeignKey( CrontabSchedule, on_delete=models.SET_NULL, @@ -371,7 +374,9 @@ class Meta: fields=["playbook_to_execute", "finished_analysis_time", "user"], name="PlaybookConfigOrdering", ), - models.Index(fields=["sent_to_bi", "-received_request_time"], name="JobBISearch"), + models.Index( + fields=["sent_to_bi", "-received_request_time"], name="JobBISearch" + ), # SELECT COUNT(*) AS "__count" FROM "api_app_job" # WHERE ("api_app_job"."depth" >= ? AND "api_app_job"."path"::text LIKE ? AND NOT ("api_app_job"."id" = ?)) models.Index(fields=["depth", "path", "id"], name="MPNodeSearch"), @@ -394,9 +399,13 @@ class Meta: null=True, # for backwards compatibility ) - analyzable = models.ForeignKey(Analyzable, related_name="jobs", on_delete=models.CASCADE) + analyzable = models.ForeignKey( + Analyzable, related_name="jobs", on_delete=models.CASCADE + ) - status = models.CharField(max_length=32, blank=False, choices=STATUSES.choices, default="pending") + status = models.CharField( + max_length=32, blank=False, choices=STATUSES.choices, default="pending" + ) analyzers_requested = models.ManyToManyField( "analyzers_manager.AnalyzerConfig", related_name="requested_in_jobs", blank=True @@ -444,8 +453,12 @@ class Meta: finished_analysis_time = models.DateTimeField(blank=True, null=True) process_time = models.FloatField(blank=True, null=True) tlp = models.CharField(max_length=8, choices=TLP.choices, default=TLP.CLEAR) - errors = pg_fields.ArrayField(models.CharField(max_length=900), blank=True, default=list, null=True) - warnings = pg_fields.ArrayField(models.TextField(), blank=True, default=list, null=True) + errors = pg_fields.ArrayField( + models.CharField(max_length=900), blank=True, default=list, null=True + ) + warnings = pg_fields.ArrayField( + models.TextField(), blank=True, default=list, null=True + ) tags = models.ManyToManyField(Tag, related_name="jobs", blank=True) scan_mode = models.IntegerField( @@ -454,7 +467,9 @@ class Meta: blank=False, default=ScanMode.CHECK_PREVIOUS_ANALYSIS.value, ) - scan_check_time = models.DurationField(null=True, blank=True, default=datetime.timedelta(hours=24)) + scan_check_time = models.DurationField( + null=True, blank=True, default=datetime.timedelta(hours=24) + ) sent_to_bi = models.BooleanField(editable=False, default=False) data_model_content_type = models.ForeignKey( ContentType, @@ -487,7 +502,12 @@ def get_root(self): # exist due to a race condition. # Note: The root cause is in django-treebeard's tree modification # operations, not in this read operation. - root_node = type(self).objects.filter(path=self.path[: self.steplen]).order_by("pk").first() + root_node = ( + type(self) + .objects.filter(path=self.path[: self.steplen]) + .order_by("pk") + .first() + ) logger.warning( f"Tree Integrity Error: Multiple roots found for Job {self.pk} " f"(path: {self.path}). Returning deterministic root " @@ -585,10 +605,16 @@ def set_final_status(self) -> None: def __get_config_reports(self, config: typing.Type["AbstractConfig"]) -> QuerySet: return getattr(self, f"{config.__name__.split('Config')[0].lower()}reports") - def __get_config_to_execute(self, config: typing.Type["AbstractConfig"]) -> QuerySet: - return getattr(self, f"{config.__name__.split('Config')[0].lower()}s_to_execute") + def __get_config_to_execute( + self, config: typing.Type["AbstractConfig"] + ) -> QuerySet: + return getattr( + self, f"{config.__name__.split('Config')[0].lower()}s_to_execute" + ) - def __get_single_config_reports_stats(self, config: typing.Type["AbstractConfig"]) -> typing.Dict: + def __get_single_config_reports_stats( + self, config: typing.Type["AbstractConfig"] + ) -> typing.Dict: reports = self.__get_config_reports(config) aggregators = { s.lower(): models.Count("status", filter=models.Q(status=s)) @@ -609,7 +635,8 @@ def _get_config_reports_stats(self) -> typing.Dict: partial_result = self.__get_single_config_reports_stats(config) # merge them result = { - k: result.get(k, 0) + partial_result.get(k, 0) for k in set(result) | set(partial_result) + k: result.get(k, 0) + partial_result.get(k, 0) + for k in set(result) | set(partial_result) } return result @@ -641,7 +668,11 @@ def kill_if_ongoing(self): def _get_signatures(self, queryset: PythonConfigQuerySet) -> Signature: config_class: PythonConfig = queryset.model - signatures = list(queryset.annotate_runnable(self.user).filter(runnable=True).get_signatures(self)) + signatures = list( + queryset.annotate_runnable(self.user) + .filter(runnable=True) + .get_signatures(self) + ) logger.info(f"{config_class} signatures are {signatures}") return ( @@ -697,13 +728,17 @@ def _get_pipeline( visualizers: PythonConfigQuerySet, ) -> Signature: runner = self._get_signatures(analyzers.distinct()) - pivots_analyzers = pivots.filter(related_analyzer_configs__isnull=False).distinct() + pivots_analyzers = pivots.filter( + related_analyzer_configs__isnull=False + ).distinct() if pivots_analyzers.exists(): runner |= self._get_signatures(pivots_analyzers) runner |= self._get_engine_signature() if connectors.exists(): runner |= self._get_signatures(connectors) - pivots_connectors = pivots.filter(related_connector_configs__isnull=False).distinct() + pivots_connectors = pivots.filter( + related_connector_configs__isnull=False + ).distinct() if pivots_connectors.exists(): runner |= self._get_signatures(pivots_connectors) if visualizers.exists(): @@ -738,7 +773,9 @@ def get_config_runtime_configuration(self, config: "AbstractConfig") -> typing.D raise TypeError( f"{config.__class__.__name__} {config.name} is not configured inside job {self.pk}" ) - return self.runtime_configuration.get(config.runtime_configuration_key, {}).get(config.name, {}) + return self.runtime_configuration.get(config.runtime_configuration_key, {}).get( + config.name, {} + ) # user methods @@ -766,12 +803,18 @@ def clean(self) -> None: self.clean_scan() def clean_scan(self): - if self.scan_mode == ScanMode.FORCE_NEW_ANALYSIS.value and self.scan_check_time is not None: + if ( + self.scan_mode == ScanMode.FORCE_NEW_ANALYSIS.value + and self.scan_check_time is not None + ): raise ValidationError( f"You can't have set mode to {ScanMode.FORCE_NEW_ANALYSIS.name}" f" and have check_time set to {self.scan_check_time}" ) - elif self.scan_mode == ScanMode.CHECK_PREVIOUS_ANALYSIS.value and self.scan_check_time is None: + elif ( + self.scan_mode == ScanMode.CHECK_PREVIOUS_ANALYSIS.value + and self.scan_check_time is None + ): raise ValidationError( f"You can't have set mode to {ScanMode.CHECK_PREVIOUS_ANALYSIS.name}" " and not have check_time set" @@ -794,11 +837,15 @@ class Parameter(models.Model): objects = ParameterQuerySet.as_manager() name = models.CharField(null=False, blank=False, max_length=50) - type = models.CharField(choices=ParamTypes.choices, max_length=10, null=False, blank=False) + type = models.CharField( + choices=ParamTypes.choices, max_length=10, null=False, blank=False + ) description = models.TextField(blank=True, default="") is_secret = models.BooleanField(db_index=True) required = models.BooleanField(null=False) - python_module = models.ForeignKey(PythonModule, related_name="parameters", on_delete=models.CASCADE) + python_module = models.ForeignKey( + PythonModule, related_name="parameters", on_delete=models.CASCADE + ) class Meta: unique_together = [["name", "python_module"]] @@ -812,7 +859,9 @@ def refresh_cache_keys(self): Refreshes the cache keys associated with the parameter's configuration class. """ self.config_class.delete_class_cache_keys() - for config in self.config_class.objects.filter(python_module=self.python_module): + for config in self.config_class.objects.filter( + python_module=self.python_module + ): config: PythonConfig config.refresh_cache_keys() @@ -845,7 +894,9 @@ class PluginConfig(OwnershipAbstractModel): objects = PluginConfigQuerySet.as_manager() value = models.JSONField(blank=True, null=True) - parameter = models.ForeignKey(Parameter, on_delete=models.CASCADE, null=False, related_name="values") + parameter = models.ForeignKey( + Parameter, on_delete=models.CASCADE, null=False, related_name="values" + ) updated_at = models.DateTimeField(auto_now=True) analyzer_config = models.ForeignKey( "analyzers_manager.AnalyzerConfig", @@ -940,7 +991,9 @@ def refresh_cache_keys(self): return if self.owner: if self.owner.has_membership() and self.owner.membership.is_admin: - for user in User.objects.filter(membership__organization=self.owner.membership.organization): + for user in User.objects.filter( + membership__organization=self.owner.membership.organization + ): self.config.delete_class_cache_keys(user) self.config.refresh_cache_keys(user) else: @@ -972,7 +1025,9 @@ def clean_config(self) -> None: Ensures that exactly one configuration type is set for this PluginConfig. """ if len(list(filter(None, self._possible_configs()))) != 1: - configs = ", ".join([config.name for config in self._possible_configs() if config]) + configs = ", ".join( + [config.name for config in self._possible_configs() if config] + ) if not configs: raise ValidationError("You must select a plugin configuration") raise ValidationError(f"You must have exactly one between {configs}") @@ -1019,6 +1074,31 @@ def is_secret(self): """Returns whether the parameter is marked as secret.""" return self.parameter.is_secret + @staticmethod + def _encrypt_value(value): + """Fernet-encrypt a value (serialized as JSON).""" + from cryptography.fernet import Fernet + + f = Fernet(settings.PLUGIN_CONFIG_FERNET_KEY) + return f.encrypt(json.dumps(value).encode()).decode() + + @staticmethod + def _decrypt_value(encrypted_value): + """Fernet-decrypt a value back to its original Python object.""" + from cryptography.fernet import Fernet + + f = Fernet(settings.PLUGIN_CONFIG_FERNET_KEY) + return json.loads(f.decrypt(encrypted_value.encode()).decode()) + + def save(self, *args, **kwargs): + if ( + self.is_secret() + and self.value is not None + and not (isinstance(self.value, str) and self.value.startswith("gAAAAA")) + ): + self.value = self._encrypt_value(self.value) + super().save(*args, **kwargs) + @property def plugin_name(self): """Returns the name of the plugin associated with this configuration.""" @@ -1084,7 +1164,9 @@ def disable_for_rate_limit(self): "Will be enabled back at " f"{enabled_to.strftime('%d %m %Y: %H %M %S')}" ) - clock_schedule = ClockedSchedule.objects.get_or_create(clocked_time=enabled_to)[0] + clock_schedule = ClockedSchedule.objects.get_or_create(clocked_time=enabled_to)[ + 0 + ] if not self.rate_limit_enable_task: from intel_owl.tasks import enable_configuration_for_org_for_rate_limit @@ -1115,7 +1197,9 @@ def disable_manually(self, user: User): user (User): The user who disabled the configuration. """ self.disabled = True - self.disabled_comment = f"Disabled by user {user.username} at {now().strftime('%Y-%m-%d %H:%M:%S')}" + self.disabled_comment = ( + f"Disabled by user {user.username} at {now().strftime('%Y-%m-%d %H:%M:%S')}" + ) if self.rate_limit_enable_task: self.rate_limit_enable_task.delete() self.save() @@ -1209,7 +1293,9 @@ def __str__(self): """Returns the name of the configuration.""" return self.name - def get_or_create_org_configuration(self, organization: Organization) -> OrganizationPluginConfiguration: + def get_or_create_org_configuration( + self, organization: Organization + ) -> OrganizationPluginConfiguration: """ Retrieves or creates the organization-specific configuration. @@ -1235,7 +1321,9 @@ def get_content_type(cls) -> ContentType: Returns: ContentType: The content type. """ - return ContentType.objects.get(model=cls._meta.model_name, app_label=cls._meta.app_label) + return ContentType.objects.get( + model=cls._meta.model_name, app_label=cls._meta.app_label + ) @property def disabled_in_organizations(self) -> QuerySet: @@ -1271,7 +1359,12 @@ def snake_case_name(cls) -> str: @deprecated("Please use `runnable` method on queryset") def is_runnable(self, user: User = None) -> bool: - return self.__class__.objects.filter(pk=self.pk).annotate_runnable(user).first().runnable + return ( + self.__class__.objects.filter(pk=self.pk) + .annotate_runnable(user) + .first() + .runnable + ) def enabled_for_user(self, user: User) -> bool: """ @@ -1316,18 +1409,26 @@ class AbstractReport(models.Model): # fields status = models.CharField(max_length=50, choices=STATUSES.choices) report = models.JSONField(default=dict) - errors = pg_fields.ArrayField(models.CharField(max_length=512), default=list, blank=True) + errors = pg_fields.ArrayField( + models.CharField(max_length=512), default=list, blank=True + ) start_time = models.DateTimeField(default=timezone.now) end_time = models.DateTimeField(default=timezone.now) task_id = models.UUIDField() # tracks celery task id - job = models.ForeignKey("api_app.Job", related_name="%(class)ss", on_delete=models.CASCADE) + job = models.ForeignKey( + "api_app.Job", related_name="%(class)ss", on_delete=models.CASCADE + ) parameters = models.JSONField(blank=False, null=False, editable=False) sent_to_bi = models.BooleanField(default=False, editable=False) class Meta: abstract = True - indexes = [models.Index(fields=["sent_to_bi", "-start_time"], name="%(class)ssBISearch")] + indexes = [ + models.Index( + fields=["sent_to_bi", "-start_time"], name="%(class)ssBISearch" + ) + ] def __str__(self): """Returns a string representation of the report.""" @@ -1375,7 +1476,9 @@ def process_time(self) -> float: secs = (self.end_time - self.start_time).total_seconds() return round(secs, 2) - def get_value(self, search_from: typing.Any, fields: typing.List[str]) -> typing.Any: + def get_value( + self, search_from: typing.Any, fields: typing.List[str] + ) -> typing.Any: if not fields: return search_from search_keyword = fields.pop(0) @@ -1394,7 +1497,9 @@ def get_value(self, search_from: typing.Any, fields: typing.List[str]) -> typing else: result.append(res) except KeyError: - errors.append(f"Field {search_keyword} not available at position {i}") + errors.append( + f"Field {search_keyword} not available at position {i}" + ) if result: self.errors.extend(errors) else: @@ -1422,7 +1527,9 @@ class PythonConfig(AbstractConfig): objects = PythonConfigQuerySet.as_manager() soft_time_limit = models.IntegerField(default=60, validators=[MinValueValidator(0)]) routing_key = models.CharField(max_length=50, default="default") - python_module = models.ForeignKey(PythonModule, on_delete=models.PROTECT, related_name="%(class)ss") + python_module = models.ForeignKey( + PythonModule, on_delete=models.PROTECT, related_name="%(class)ss" + ) health_check_task = models.OneToOneField( PeriodicTask, @@ -1540,7 +1647,9 @@ def generate_empty_report(self, job: Job, task_id: str, status: str): "task_id": task_id, "start_time": now(), "end_time": now(), - "parameters": self._get_params(job.user, job.get_config_runtime_configuration(self)), + "parameters": self._get_params( + job.user, job.get_config_runtime_configuration(self) + ), }, )[0] @@ -1553,19 +1662,21 @@ def refresh_cache_keys(self, user: User = None): """ from api_app.serializers.plugin import PythonConfigListSerializer - base_key = f"{self.__class__.__name__}_{self.name}_{user.username if user else ''}" + base_key = ( + f"{self.__class__.__name__}_{self.name}_{user.username if user else ''}" + ) for key in cache.get_where(f"serializer_{base_key}").keys(): logger.debug(f"Deleting cache key {key}") cache.delete(key) if user: - PythonConfigListSerializer(child=self.serializer_class()).to_representation_single_plugin( - self, user - ) + PythonConfigListSerializer( + child=self.serializer_class() + ).to_representation_single_plugin(self, user) else: for generic_user in User.objects.exclude(email=""): - PythonConfigListSerializer(child=self.serializer_class()).to_representation_single_plugin( - self, generic_user - ) + PythonConfigListSerializer( + child=self.serializer_class() + ).to_representation_single_plugin(self, generic_user) @classproperty def serializer_class(cls) -> Type["PythonConfigSerializer"]: @@ -1702,7 +1813,9 @@ def config_exception(cls): """ raise NotImplementedError() - def read_configured_params(self, user: User = None, config_runtime: Dict = None) -> ParameterQuerySet: + def read_configured_params( + self, user: User = None, config_runtime: Dict = None + ) -> ParameterQuerySet: """ Reads the configured parameters for the plugin. @@ -1713,13 +1826,15 @@ def read_configured_params(self, user: User = None, config_runtime: Dict = None) Returns: ParameterQuerySet: The queryset of configured parameters. """ - params = self.parameters.annotate_configured(self, user).annotate_value_for_user( - self, user, config_runtime - ) + params = self.parameters.annotate_configured( + self, user + ).annotate_value_for_user(self, user, config_runtime) # Use a single .first() instead of .exists() + .first() to reduce # DB queries from 2 to 1. select_related avoids a lazy‑load query # when we access param.python_module.module in the error message. - not_configured_params = params.filter(required=True, configured=False).select_related("python_module") + not_configured_params = params.filter( + required=True, configured=False + ).select_related("python_module") param = not_configured_params.first() if param is not None: @@ -1743,7 +1858,10 @@ def generate_health_check_periodic_task(self): """ from intel_owl.tasks import health_check - if hasattr(self.python_module, "health_check_schedule") and self.python_module.health_check_schedule: + if ( + hasattr(self.python_module, "health_check_schedule") + and self.python_module.health_check_schedule + ): periodic_task = PeriodicTask.objects.update_or_create( name__iexact=f"{self.name}HealthCheck{self.__class__.__name__}", task=f"{health_check.__module__}.{health_check.__name__}", diff --git a/intel_owl/settings/security.py b/intel_owl/settings/security.py index 8be1ea5172..d0e79a3b47 100644 --- a/intel_owl/settings/security.py +++ b/intel_owl/settings/security.py @@ -2,6 +2,9 @@ # See the file 'LICENSE' for copying permission. # Security Stuff +import base64 +import hashlib + from django.core.management.utils import get_random_secret_key from ._util import get_secret @@ -26,6 +29,13 @@ CSRF_TRUSTED_ORIGINS = [f"{WEB_CLIENT_URL}:80/"] ALLOWED_HOSTS = ["*"] +# Fernet key for encrypting plugin secrets at rest. +# Falls back to SECRET_KEY if PLUGIN_CONFIG_SECRET_KEY is not set. +_raw_secret = get_secret("PLUGIN_CONFIG_SECRET_KEY", SECRET_KEY) +PLUGIN_CONFIG_FERNET_KEY = base64.urlsafe_b64encode( + hashlib.sha256(_raw_secret.encode()).digest() +) + # https://docs.djangoproject.com/en/4.2/ref/settings/#data-upload-max-memory-size DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * (10**6) FILE_UPLOAD_MAX_MEMORY_SIZE = 100 * (10**6) diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index a2eeaaada9..c747ce0333 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -118,3 +118,4 @@ DeepDiff==8.6.1 lxml==6.0.2 Faker==36.1.0 beautifulsoup4==4.14.2 +cryptography==46.0.0 diff --git a/tests/api_app/test_plugin_config_encryption.py b/tests/api_app/test_plugin_config_encryption.py new file mode 100644 index 0000000000..2bdd595318 --- /dev/null +++ b/tests/api_app/test_plugin_config_encryption.py @@ -0,0 +1,103 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from api_app.choices import PythonModuleBasePaths +from api_app.models import Parameter, PluginConfig, PythonModule +from api_app.visualizers_manager.models import VisualizerConfig +from tests import CustomTestCase + + +class PluginConfigEncryptionTestCase(CustomTestCase): + + def setUp(self): + super().setUp() + self.vc, _ = VisualizerConfig.objects.get_or_create( + name="test_encryption", + description="test encryption", + python_module=PythonModule.objects.get( + base_path=PythonModuleBasePaths.Visualizer.value, + module="yara.Yara", + ), + disabled=False, + ) + self.secret_param = Parameter.objects.create( + python_module=self.vc.python_module, + name="test_api_key", + type="str", + is_secret=True, + required=False, + ) + self.non_secret_param = Parameter.objects.create( + python_module=self.vc.python_module, + name="test_max_retries", + type="int", + is_secret=False, + required=False, + ) + + def tearDown(self): + self.secret_param.delete() + self.non_secret_param.delete() + self.vc.delete() + super().tearDown() + + def test_secret_value_encrypted_on_save(self): + pc = PluginConfig.objects.create( + owner=self.user, + for_organization=False, + parameter=self.secret_param, + value="my_super_secret_api_key_12345", + visualizer_config=self.vc, + ) + pc.refresh_from_db() + self.assertIsInstance(pc.value, str) + self.assertTrue(pc.value.startswith("gAAAAA")) + pc.delete() + + def test_encrypt_decrypt_roundtrip(self): + original = "my_super_secret_api_key_12345" + encrypted = PluginConfig._encrypt_value(original) + self.assertTrue(encrypted.startswith("gAAAAA")) + self.assertEqual(PluginConfig._decrypt_value(encrypted), original) + + def test_non_secret_value_unchanged(self): + pc = PluginConfig.objects.create( + owner=self.user, + for_organization=False, + parameter=self.non_secret_param, + value=10, + visualizer_config=self.vc, + ) + pc.refresh_from_db() + self.assertEqual(pc.value, 10) + pc.delete() + + def test_no_double_encryption(self): + pc = PluginConfig.objects.create( + owner=self.user, + for_organization=False, + parameter=self.secret_param, + value="test_secret_value", + visualizer_config=self.vc, + ) + pc.refresh_from_db() + first_encrypted = pc.value + + # saving again should not re-encrypt + pc.save() + pc.refresh_from_db() + self.assertEqual(pc.value, first_encrypted) + self.assertEqual(PluginConfig._decrypt_value(pc.value), "test_secret_value") + pc.delete() + + def test_encrypt_decrypt_dict(self): + original = {"key": "value", "nested": {"a": 1}} + encrypted = PluginConfig._encrypt_value(original) + self.assertTrue(encrypted.startswith("gAAAAA")) + self.assertEqual(PluginConfig._decrypt_value(encrypted), original) + + def test_encrypt_decrypt_list(self): + original = ["secret1", "secret2", "secret3"] + encrypted = PluginConfig._encrypt_value(original) + self.assertTrue(encrypted.startswith("gAAAAA")) + self.assertEqual(PluginConfig._decrypt_value(encrypted), original) From a57cc5adee5b722c1ee89d4132dc8bfd9f4de1fa Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 24 Mar 2026 01:44:25 +0530 Subject: [PATCH 2/5] fix: ruff import sorting and upgrade cryptography to 46.0.5 --- api_app/classes.py | 2 +- api_app/models.py | 5 ++--- requirements/project-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/api_app/classes.py b/api_app/classes.py index 0161f3cc69..678f89480f 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -7,7 +7,6 @@ import requests from billiard.exceptions import SoftTimeLimitExceeded -from certego_saas.apps.user.models import User from django.conf import settings from django.core.files import File from django.utils import timezone @@ -16,6 +15,7 @@ from api_app.decorators import abstractclassproperty, classproperty from api_app.models import AbstractReport, Job, PythonConfig, PythonModule +from certego_saas.apps.user.models import User logger = logging.getLogger(__name__) diff --git a/api_app/models.py b/api_app/models.py index c5236901f6..5651f76efd 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -48,9 +48,6 @@ if typing.TYPE_CHECKING: from api_app.classes import Plugin -from certego_saas.apps.organization.organization import Organization -from certego_saas.models import User - from api_app.decorators import classproperty from api_app.defaults import default_runtime from api_app.helpers import deprecated, get_now @@ -65,6 +62,8 @@ PythonConfigQuerySet, ) from api_app.validators import plugin_name_validator, validate_runtime_configuration +from certego_saas.apps.organization.organization import Organization +from certego_saas.models import User from intel_owl import tasks from intel_owl.celery import get_queue_name diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index c747ce0333..0da6544aaa 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -118,4 +118,4 @@ DeepDiff==8.6.1 lxml==6.0.2 Faker==36.1.0 beautifulsoup4==4.14.2 -cryptography==46.0.0 +cryptography==46.0.5 From 8033f8a74625876701f83308ee6a67c3356c393b Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 24 Mar 2026 01:58:15 +0530 Subject: [PATCH 3/5] style: apply ruff format --- api_app/classes.py | 38 +--- api_app/models.py | 190 +++++------------- intel_owl/settings/security.py | 4 +- .../api_app/test_plugin_config_encryption.py | 1 - 4 files changed, 59 insertions(+), 174 deletions(-) diff --git a/api_app/classes.py b/api_app/classes.py index 678f89480f..3a4cf7cfab 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -140,28 +140,18 @@ def __str__(self): return f"{self.__class__.__name__}" def config(self, runtime_configuration: typing.Dict): - self.__parameters = self._config.read_configured_params( - self._user, runtime_configuration - ) + self.__parameters = self._config.read_configured_params(self._user, runtime_configuration) for parameter in self.__parameters: - attribute_name = ( - f"_{parameter.name}" if parameter.is_secret else parameter.name - ) + attribute_name = f"_{parameter.name}" if parameter.is_secret else parameter.name value = parameter.value # decrypt secrets that were stored encrypted - if ( - parameter.is_secret - and isinstance(value, str) - and value.startswith("gAAAAA") - ): + if parameter.is_secret and isinstance(value, str) and value.startswith("gAAAAA"): from api_app.models import PluginConfig value = PluginConfig._decrypt_value(value) setattr(self, attribute_name, value) logger.debug( - f"Adding to {self.__class__.__name__} " - f"param {attribute_name} " - f"with value {parameter.value} " + f"Adding to {self.__class__.__name__} param {attribute_name} with value {parameter.value} " ) def before_run(self): @@ -216,9 +206,7 @@ def log_error(self, e): Args: e (Exception): The exception to log. """ - if isinstance( - e, (*self.get_exceptions_to_catch(), SoftTimeLimitExceeded, HTTPError) - ): + if isinstance(e, (*self.get_exceptions_to_catch(), SoftTimeLimitExceeded, HTTPError)): error_message = self.get_error_message(e) logger.error(error_message) else: @@ -237,9 +225,7 @@ def after_run_failed(self, e: Exception): self.report.status = self.report.STATUSES.FAILED self.report.save(update_fields=["status", "errors"]) if isinstance(e, HTTPError) and ( - hasattr(e, "response") - and hasattr(e.response, "status_code") - and e.response.status_code == 429 + hasattr(e, "response") and hasattr(e.response, "status_code") and e.response.status_code == 429 ): self.disable_for_rate_limit() else: @@ -279,9 +265,7 @@ def get_error_message(self, err, is_base_err=False): f" '{err}'" ) - def start( - self, job_id: int, runtime_configuration: dict, task_id: str, *args, **kwargs - ): + def start(self, job_id: int, runtime_configuration: dict, task_id: str, *args, **kwargs): """ Entrypoint function to execute the plugin. calls `before_run`, `run`, `after_run` @@ -419,9 +403,7 @@ def disable_for_rate_limit(self): self._user.membership.organization ) if org_configuration.rate_limit_timeout is not None: - api_key_parameter = self.__parameters.filter( - name__contains="api_key" - ).first() + api_key_parameter = self.__parameters.filter(name__contains="api_key").first() # if we do not have api keys OR the api key was org based # OR if the api key is not actually required and we do not have it set if ( @@ -431,9 +413,7 @@ def disable_for_rate_limit(self): ): org_configuration.disable_for_rate_limit() else: - logger.warning( - f"Not disabling {self} because api key used is personal" - ) + logger.warning(f"Not disabling {self} because api key used is personal") else: logger.warning( f"You are trying to disable {self} for rate limit without specifying a timeout." diff --git a/api_app/models.py b/api_app/models.py index 5651f76efd..3161a86d8f 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -116,9 +116,7 @@ class PythonModule(models.Model): """ module = models.CharField(max_length=120, db_index=True) - base_path = models.CharField( - max_length=120, db_index=True, choices=PythonModuleBasePaths.choices - ) + base_path = models.CharField(max_length=120, db_index=True, choices=PythonModuleBasePaths.choices) update_schedule = models.ForeignKey( CrontabSchedule, on_delete=models.SET_NULL, @@ -373,9 +371,7 @@ class Meta: fields=["playbook_to_execute", "finished_analysis_time", "user"], name="PlaybookConfigOrdering", ), - models.Index( - fields=["sent_to_bi", "-received_request_time"], name="JobBISearch" - ), + models.Index(fields=["sent_to_bi", "-received_request_time"], name="JobBISearch"), # SELECT COUNT(*) AS "__count" FROM "api_app_job" # WHERE ("api_app_job"."depth" >= ? AND "api_app_job"."path"::text LIKE ? AND NOT ("api_app_job"."id" = ?)) models.Index(fields=["depth", "path", "id"], name="MPNodeSearch"), @@ -398,13 +394,9 @@ class Meta: null=True, # for backwards compatibility ) - analyzable = models.ForeignKey( - Analyzable, related_name="jobs", on_delete=models.CASCADE - ) + analyzable = models.ForeignKey(Analyzable, related_name="jobs", on_delete=models.CASCADE) - status = models.CharField( - max_length=32, blank=False, choices=STATUSES.choices, default="pending" - ) + status = models.CharField(max_length=32, blank=False, choices=STATUSES.choices, default="pending") analyzers_requested = models.ManyToManyField( "analyzers_manager.AnalyzerConfig", related_name="requested_in_jobs", blank=True @@ -452,12 +444,8 @@ class Meta: finished_analysis_time = models.DateTimeField(blank=True, null=True) process_time = models.FloatField(blank=True, null=True) tlp = models.CharField(max_length=8, choices=TLP.choices, default=TLP.CLEAR) - errors = pg_fields.ArrayField( - models.CharField(max_length=900), blank=True, default=list, null=True - ) - warnings = pg_fields.ArrayField( - models.TextField(), blank=True, default=list, null=True - ) + errors = pg_fields.ArrayField(models.CharField(max_length=900), blank=True, default=list, null=True) + warnings = pg_fields.ArrayField(models.TextField(), blank=True, default=list, null=True) tags = models.ManyToManyField(Tag, related_name="jobs", blank=True) scan_mode = models.IntegerField( @@ -466,9 +454,7 @@ class Meta: blank=False, default=ScanMode.CHECK_PREVIOUS_ANALYSIS.value, ) - scan_check_time = models.DurationField( - null=True, blank=True, default=datetime.timedelta(hours=24) - ) + scan_check_time = models.DurationField(null=True, blank=True, default=datetime.timedelta(hours=24)) sent_to_bi = models.BooleanField(editable=False, default=False) data_model_content_type = models.ForeignKey( ContentType, @@ -501,12 +487,7 @@ def get_root(self): # exist due to a race condition. # Note: The root cause is in django-treebeard's tree modification # operations, not in this read operation. - root_node = ( - type(self) - .objects.filter(path=self.path[: self.steplen]) - .order_by("pk") - .first() - ) + root_node = type(self).objects.filter(path=self.path[: self.steplen]).order_by("pk").first() logger.warning( f"Tree Integrity Error: Multiple roots found for Job {self.pk} " f"(path: {self.path}). Returning deterministic root " @@ -604,16 +585,10 @@ def set_final_status(self) -> None: def __get_config_reports(self, config: typing.Type["AbstractConfig"]) -> QuerySet: return getattr(self, f"{config.__name__.split('Config')[0].lower()}reports") - def __get_config_to_execute( - self, config: typing.Type["AbstractConfig"] - ) -> QuerySet: - return getattr( - self, f"{config.__name__.split('Config')[0].lower()}s_to_execute" - ) + def __get_config_to_execute(self, config: typing.Type["AbstractConfig"]) -> QuerySet: + return getattr(self, f"{config.__name__.split('Config')[0].lower()}s_to_execute") - def __get_single_config_reports_stats( - self, config: typing.Type["AbstractConfig"] - ) -> typing.Dict: + def __get_single_config_reports_stats(self, config: typing.Type["AbstractConfig"]) -> typing.Dict: reports = self.__get_config_reports(config) aggregators = { s.lower(): models.Count("status", filter=models.Q(status=s)) @@ -634,8 +609,7 @@ def _get_config_reports_stats(self) -> typing.Dict: partial_result = self.__get_single_config_reports_stats(config) # merge them result = { - k: result.get(k, 0) + partial_result.get(k, 0) - for k in set(result) | set(partial_result) + k: result.get(k, 0) + partial_result.get(k, 0) for k in set(result) | set(partial_result) } return result @@ -667,11 +641,7 @@ def kill_if_ongoing(self): def _get_signatures(self, queryset: PythonConfigQuerySet) -> Signature: config_class: PythonConfig = queryset.model - signatures = list( - queryset.annotate_runnable(self.user) - .filter(runnable=True) - .get_signatures(self) - ) + signatures = list(queryset.annotate_runnable(self.user).filter(runnable=True).get_signatures(self)) logger.info(f"{config_class} signatures are {signatures}") return ( @@ -727,17 +697,13 @@ def _get_pipeline( visualizers: PythonConfigQuerySet, ) -> Signature: runner = self._get_signatures(analyzers.distinct()) - pivots_analyzers = pivots.filter( - related_analyzer_configs__isnull=False - ).distinct() + pivots_analyzers = pivots.filter(related_analyzer_configs__isnull=False).distinct() if pivots_analyzers.exists(): runner |= self._get_signatures(pivots_analyzers) runner |= self._get_engine_signature() if connectors.exists(): runner |= self._get_signatures(connectors) - pivots_connectors = pivots.filter( - related_connector_configs__isnull=False - ).distinct() + pivots_connectors = pivots.filter(related_connector_configs__isnull=False).distinct() if pivots_connectors.exists(): runner |= self._get_signatures(pivots_connectors) if visualizers.exists(): @@ -772,9 +738,7 @@ def get_config_runtime_configuration(self, config: "AbstractConfig") -> typing.D raise TypeError( f"{config.__class__.__name__} {config.name} is not configured inside job {self.pk}" ) - return self.runtime_configuration.get(config.runtime_configuration_key, {}).get( - config.name, {} - ) + return self.runtime_configuration.get(config.runtime_configuration_key, {}).get(config.name, {}) # user methods @@ -802,18 +766,12 @@ def clean(self) -> None: self.clean_scan() def clean_scan(self): - if ( - self.scan_mode == ScanMode.FORCE_NEW_ANALYSIS.value - and self.scan_check_time is not None - ): + if self.scan_mode == ScanMode.FORCE_NEW_ANALYSIS.value and self.scan_check_time is not None: raise ValidationError( f"You can't have set mode to {ScanMode.FORCE_NEW_ANALYSIS.name}" f" and have check_time set to {self.scan_check_time}" ) - elif ( - self.scan_mode == ScanMode.CHECK_PREVIOUS_ANALYSIS.value - and self.scan_check_time is None - ): + elif self.scan_mode == ScanMode.CHECK_PREVIOUS_ANALYSIS.value and self.scan_check_time is None: raise ValidationError( f"You can't have set mode to {ScanMode.CHECK_PREVIOUS_ANALYSIS.name}" " and not have check_time set" @@ -836,15 +794,11 @@ class Parameter(models.Model): objects = ParameterQuerySet.as_manager() name = models.CharField(null=False, blank=False, max_length=50) - type = models.CharField( - choices=ParamTypes.choices, max_length=10, null=False, blank=False - ) + type = models.CharField(choices=ParamTypes.choices, max_length=10, null=False, blank=False) description = models.TextField(blank=True, default="") is_secret = models.BooleanField(db_index=True) required = models.BooleanField(null=False) - python_module = models.ForeignKey( - PythonModule, related_name="parameters", on_delete=models.CASCADE - ) + python_module = models.ForeignKey(PythonModule, related_name="parameters", on_delete=models.CASCADE) class Meta: unique_together = [["name", "python_module"]] @@ -858,9 +812,7 @@ def refresh_cache_keys(self): Refreshes the cache keys associated with the parameter's configuration class. """ self.config_class.delete_class_cache_keys() - for config in self.config_class.objects.filter( - python_module=self.python_module - ): + for config in self.config_class.objects.filter(python_module=self.python_module): config: PythonConfig config.refresh_cache_keys() @@ -893,9 +845,7 @@ class PluginConfig(OwnershipAbstractModel): objects = PluginConfigQuerySet.as_manager() value = models.JSONField(blank=True, null=True) - parameter = models.ForeignKey( - Parameter, on_delete=models.CASCADE, null=False, related_name="values" - ) + parameter = models.ForeignKey(Parameter, on_delete=models.CASCADE, null=False, related_name="values") updated_at = models.DateTimeField(auto_now=True) analyzer_config = models.ForeignKey( "analyzers_manager.AnalyzerConfig", @@ -990,9 +940,7 @@ def refresh_cache_keys(self): return if self.owner: if self.owner.has_membership() and self.owner.membership.is_admin: - for user in User.objects.filter( - membership__organization=self.owner.membership.organization - ): + for user in User.objects.filter(membership__organization=self.owner.membership.organization): self.config.delete_class_cache_keys(user) self.config.refresh_cache_keys(user) else: @@ -1024,9 +972,7 @@ def clean_config(self) -> None: Ensures that exactly one configuration type is set for this PluginConfig. """ if len(list(filter(None, self._possible_configs()))) != 1: - configs = ", ".join( - [config.name for config in self._possible_configs() if config] - ) + configs = ", ".join([config.name for config in self._possible_configs() if config]) if not configs: raise ValidationError("You must select a plugin configuration") raise ValidationError(f"You must have exactly one between {configs}") @@ -1163,9 +1109,7 @@ def disable_for_rate_limit(self): "Will be enabled back at " f"{enabled_to.strftime('%d %m %Y: %H %M %S')}" ) - clock_schedule = ClockedSchedule.objects.get_or_create(clocked_time=enabled_to)[ - 0 - ] + clock_schedule = ClockedSchedule.objects.get_or_create(clocked_time=enabled_to)[0] if not self.rate_limit_enable_task: from intel_owl.tasks import enable_configuration_for_org_for_rate_limit @@ -1196,9 +1140,7 @@ def disable_manually(self, user: User): user (User): The user who disabled the configuration. """ self.disabled = True - self.disabled_comment = ( - f"Disabled by user {user.username} at {now().strftime('%Y-%m-%d %H:%M:%S')}" - ) + self.disabled_comment = f"Disabled by user {user.username} at {now().strftime('%Y-%m-%d %H:%M:%S')}" if self.rate_limit_enable_task: self.rate_limit_enable_task.delete() self.save() @@ -1292,9 +1234,7 @@ def __str__(self): """Returns the name of the configuration.""" return self.name - def get_or_create_org_configuration( - self, organization: Organization - ) -> OrganizationPluginConfiguration: + def get_or_create_org_configuration(self, organization: Organization) -> OrganizationPluginConfiguration: """ Retrieves or creates the organization-specific configuration. @@ -1320,9 +1260,7 @@ def get_content_type(cls) -> ContentType: Returns: ContentType: The content type. """ - return ContentType.objects.get( - model=cls._meta.model_name, app_label=cls._meta.app_label - ) + return ContentType.objects.get(model=cls._meta.model_name, app_label=cls._meta.app_label) @property def disabled_in_organizations(self) -> QuerySet: @@ -1358,12 +1296,7 @@ def snake_case_name(cls) -> str: @deprecated("Please use `runnable` method on queryset") def is_runnable(self, user: User = None) -> bool: - return ( - self.__class__.objects.filter(pk=self.pk) - .annotate_runnable(user) - .first() - .runnable - ) + return self.__class__.objects.filter(pk=self.pk).annotate_runnable(user).first().runnable def enabled_for_user(self, user: User) -> bool: """ @@ -1408,26 +1341,18 @@ class AbstractReport(models.Model): # fields status = models.CharField(max_length=50, choices=STATUSES.choices) report = models.JSONField(default=dict) - errors = pg_fields.ArrayField( - models.CharField(max_length=512), default=list, blank=True - ) + errors = pg_fields.ArrayField(models.CharField(max_length=512), default=list, blank=True) start_time = models.DateTimeField(default=timezone.now) end_time = models.DateTimeField(default=timezone.now) task_id = models.UUIDField() # tracks celery task id - job = models.ForeignKey( - "api_app.Job", related_name="%(class)ss", on_delete=models.CASCADE - ) + job = models.ForeignKey("api_app.Job", related_name="%(class)ss", on_delete=models.CASCADE) parameters = models.JSONField(blank=False, null=False, editable=False) sent_to_bi = models.BooleanField(default=False, editable=False) class Meta: abstract = True - indexes = [ - models.Index( - fields=["sent_to_bi", "-start_time"], name="%(class)ssBISearch" - ) - ] + indexes = [models.Index(fields=["sent_to_bi", "-start_time"], name="%(class)ssBISearch")] def __str__(self): """Returns a string representation of the report.""" @@ -1475,9 +1400,7 @@ def process_time(self) -> float: secs = (self.end_time - self.start_time).total_seconds() return round(secs, 2) - def get_value( - self, search_from: typing.Any, fields: typing.List[str] - ) -> typing.Any: + def get_value(self, search_from: typing.Any, fields: typing.List[str]) -> typing.Any: if not fields: return search_from search_keyword = fields.pop(0) @@ -1496,9 +1419,7 @@ def get_value( else: result.append(res) except KeyError: - errors.append( - f"Field {search_keyword} not available at position {i}" - ) + errors.append(f"Field {search_keyword} not available at position {i}") if result: self.errors.extend(errors) else: @@ -1526,9 +1447,7 @@ class PythonConfig(AbstractConfig): objects = PythonConfigQuerySet.as_manager() soft_time_limit = models.IntegerField(default=60, validators=[MinValueValidator(0)]) routing_key = models.CharField(max_length=50, default="default") - python_module = models.ForeignKey( - PythonModule, on_delete=models.PROTECT, related_name="%(class)ss" - ) + python_module = models.ForeignKey(PythonModule, on_delete=models.PROTECT, related_name="%(class)ss") health_check_task = models.OneToOneField( PeriodicTask, @@ -1646,9 +1565,7 @@ def generate_empty_report(self, job: Job, task_id: str, status: str): "task_id": task_id, "start_time": now(), "end_time": now(), - "parameters": self._get_params( - job.user, job.get_config_runtime_configuration(self) - ), + "parameters": self._get_params(job.user, job.get_config_runtime_configuration(self)), }, )[0] @@ -1661,21 +1578,19 @@ def refresh_cache_keys(self, user: User = None): """ from api_app.serializers.plugin import PythonConfigListSerializer - base_key = ( - f"{self.__class__.__name__}_{self.name}_{user.username if user else ''}" - ) + base_key = f"{self.__class__.__name__}_{self.name}_{user.username if user else ''}" for key in cache.get_where(f"serializer_{base_key}").keys(): logger.debug(f"Deleting cache key {key}") cache.delete(key) if user: - PythonConfigListSerializer( - child=self.serializer_class() - ).to_representation_single_plugin(self, user) + PythonConfigListSerializer(child=self.serializer_class()).to_representation_single_plugin( + self, user + ) else: for generic_user in User.objects.exclude(email=""): - PythonConfigListSerializer( - child=self.serializer_class() - ).to_representation_single_plugin(self, generic_user) + PythonConfigListSerializer(child=self.serializer_class()).to_representation_single_plugin( + self, generic_user + ) @classproperty def serializer_class(cls) -> Type["PythonConfigSerializer"]: @@ -1812,9 +1727,7 @@ def config_exception(cls): """ raise NotImplementedError() - def read_configured_params( - self, user: User = None, config_runtime: Dict = None - ) -> ParameterQuerySet: + def read_configured_params(self, user: User = None, config_runtime: Dict = None) -> ParameterQuerySet: """ Reads the configured parameters for the plugin. @@ -1825,15 +1738,13 @@ def read_configured_params( Returns: ParameterQuerySet: The queryset of configured parameters. """ - params = self.parameters.annotate_configured( - self, user - ).annotate_value_for_user(self, user, config_runtime) + params = self.parameters.annotate_configured(self, user).annotate_value_for_user( + self, user, config_runtime + ) # Use a single .first() instead of .exists() + .first() to reduce # DB queries from 2 to 1. select_related avoids a lazy‑load query # when we access param.python_module.module in the error message. - not_configured_params = params.filter( - required=True, configured=False - ).select_related("python_module") + not_configured_params = params.filter(required=True, configured=False).select_related("python_module") param = not_configured_params.first() if param is not None: @@ -1857,10 +1768,7 @@ def generate_health_check_periodic_task(self): """ from intel_owl.tasks import health_check - if ( - hasattr(self.python_module, "health_check_schedule") - and self.python_module.health_check_schedule - ): + if hasattr(self.python_module, "health_check_schedule") and self.python_module.health_check_schedule: periodic_task = PeriodicTask.objects.update_or_create( name__iexact=f"{self.name}HealthCheck{self.__class__.__name__}", task=f"{health_check.__module__}.{health_check.__name__}", diff --git a/intel_owl/settings/security.py b/intel_owl/settings/security.py index d0e79a3b47..c0fbe9972a 100644 --- a/intel_owl/settings/security.py +++ b/intel_owl/settings/security.py @@ -32,9 +32,7 @@ # Fernet key for encrypting plugin secrets at rest. # Falls back to SECRET_KEY if PLUGIN_CONFIG_SECRET_KEY is not set. _raw_secret = get_secret("PLUGIN_CONFIG_SECRET_KEY", SECRET_KEY) -PLUGIN_CONFIG_FERNET_KEY = base64.urlsafe_b64encode( - hashlib.sha256(_raw_secret.encode()).digest() -) +PLUGIN_CONFIG_FERNET_KEY = base64.urlsafe_b64encode(hashlib.sha256(_raw_secret.encode()).digest()) # https://docs.djangoproject.com/en/4.2/ref/settings/#data-upload-max-memory-size DATA_UPLOAD_MAX_MEMORY_SIZE = 100 * (10**6) diff --git a/tests/api_app/test_plugin_config_encryption.py b/tests/api_app/test_plugin_config_encryption.py index 2bdd595318..c4b9ea03f7 100644 --- a/tests/api_app/test_plugin_config_encryption.py +++ b/tests/api_app/test_plugin_config_encryption.py @@ -8,7 +8,6 @@ class PluginConfigEncryptionTestCase(CustomTestCase): - def setUp(self): super().setUp() self.vc, _ = VisualizerConfig.objects.get_or_create( From f66d10003e0e250a49c12fe504f0d151b5e1227a Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 24 Mar 2026 03:40:30 +0530 Subject: [PATCH 4/5] test: update assertions to account for encrypted secret values --- tests/api_app/test_views.py | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/api_app/test_views.py b/tests/api_app/test_views.py index e78b69a32a..6774581596 100644 --- a/tests/api_app/test_views.py +++ b/tests/api_app/test_views.py @@ -678,7 +678,7 @@ def test_plugin_config(self): for config in [*org_config, *user_config]: if config["attribute"] == "mynewparameter": - self.assertEqual(config["value"], "supersecret") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret") # if the user is admin of an org, he should get the org secret self.client.force_authenticate(user=self.admin) @@ -690,7 +690,7 @@ def test_plugin_config(self): for config in [*org_config, *user_config]: if config["attribute"] == "mynewparameter": - self.assertEqual(config["value"], "supersecret") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret") # second personal item secret_owner = PluginConfig( @@ -712,10 +712,10 @@ def test_plugin_config(self): for config in org_config: if config["attribute"] == "mynewparameter": - self.assertEqual(config["value"], "supersecret") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret") for config in user_config: if config["attribute"] == "mynewparameter": - self.assertEqual(config["value"], "supersecret_user_only") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret_user_only") # other users cannot see user's personal items self.client.force_authenticate(user=self.admin) @@ -727,11 +727,11 @@ def test_plugin_config(self): for config in org_config: if config["attribute"] == "mynewparameter": - self.assertEqual(config["value"], "supersecret") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret") for config in user_config: if config["attribute"] == "mynewparameter": - self.assertNotEqual(config["value"], "supersecret_user_only") - self.assertEqual(config["value"], "supersecret") + self.assertNotEqual(PluginConfig._decrypt_value(config["value"]), "supersecret_user_only") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret") # if a standard user who does not belong to any org tries to get a secret, # they should not find anything @@ -768,7 +768,7 @@ def test_plugin_config(self): self.assertEqual(config["value"], "redacted") secret_owner.refresh_from_db() - self.assertEqual(secret_owner.value, "supersecret_user_only") + self.assertEqual(PluginConfig._decrypt_value(secret_owner.value), "supersecret_user_only") # third superuser secret secret_owner = PluginConfig( @@ -790,10 +790,10 @@ def test_plugin_config(self): self.assertEqual(config["value"], "redacted") for config in user_config: if config["attribute"] == "mynewparameter": - self.assertEqual(config["value"], "supersecret_low_privilege") + self.assertEqual(PluginConfig._decrypt_value(config["value"]), "supersecret_low_privilege") param.delete() - PluginConfig.objects.filter(value__startswith="supersecret").delete() + PluginConfig.objects.filter(value__startswith="gAAAAA").delete() org.delete() def test_plugin_config_list(self): @@ -856,7 +856,7 @@ def test_plugin_config_list(self): self.assertIn("organization", needle) self.assertEqual(needle["organization"], "testorg0") self.assertIn("value", needle) - self.assertEqual(needle["value"], "value") + self.assertEqual(PluginConfig._decrypt_value(needle["value"]), "value") self.assertIn("attribute", needle) self.assertEqual(needle["attribute"], "test") self.assertIn("required", needle) @@ -883,7 +883,7 @@ def test_plugin_config_list(self): self.assertIn("organization", needle) self.assertEqual(needle["organization"], "testorg0") self.assertIn("value", needle) - self.assertEqual(needle["value"], "value") + self.assertEqual(PluginConfig._decrypt_value(needle["value"]), "value") self.assertIn("attribute", needle) self.assertEqual(needle["attribute"], "test") self.assertIn("required", needle) @@ -998,7 +998,7 @@ def test_update(self): response = self.client.patch(uri, payload, format="json") self.assertEqual(response.status_code, 200) pc1 = PluginConfig.objects.get(id=pc.pk) - self.assertEqual(pc1.value, "new_org_supersecret") + self.assertEqual(PluginConfig._decrypt_value(pc1.value), "new_org_supersecret") # admin can update org secret self.client.force_authenticate(user=self.admin) @@ -1012,7 +1012,7 @@ def test_update(self): response = self.client.patch(uri, payload, format="json") self.assertEqual(response.status_code, 200) pc1 = PluginConfig.objects.get(id=pc.pk) - self.assertEqual(pc1.value, "new_org_supersecret_admin") + self.assertEqual(PluginConfig._decrypt_value(pc1.value), "new_org_supersecret_admin") # user can not update org secret self.client.force_authenticate(user=self.user) @@ -1049,7 +1049,7 @@ def test_update(self): response = self.client.patch(uri, payload, format="json") self.assertEqual(response.status_code, 200) pc_user = PluginConfig.objects.get(id=secret_owner.pk) - self.assertEqual(pc_user.value, "new_supersecret_user_only") + self.assertEqual(PluginConfig._decrypt_value(pc_user.value), "new_supersecret_user_only") # other users cannot update user's personal items self.client.force_authenticate(user=self.guest) @@ -1063,14 +1063,14 @@ def test_update(self): response = self.client.patch(uri, payload, format="json") self.assertEqual(response.status_code, 403) pc_user = PluginConfig.objects.get(id=secret_owner.pk) - self.assertEqual(pc_user.value, "new_supersecret_user_only") - self.assertNotEqual(pc_user.value, "new_supersecret") + self.assertEqual(PluginConfig._decrypt_value(pc_user.value), "new_supersecret_user_only") + self.assertNotEqual(PluginConfig._decrypt_value(pc_user.value), "new_supersecret") secret_owner.delete() pc.delete() param.delete() - PluginConfig.objects.filter(value__startswith="supersecret").delete() + PluginConfig.objects.filter(value__startswith="gAAAAA").delete() org.delete() def test_create(self): @@ -1121,7 +1121,7 @@ def test_create(self): response = self.client.post(uri, payload, format="json") self.assertEqual(response.status_code, 201) content = response.json() - self.assertEqual(content[0]["value"], "new_org_supersecret") + self.assertEqual(PluginConfig._decrypt_value(content[0]["value"]), "new_org_supersecret") self.assertEqual(content[0]["owner"], self.superuser.username) pc = PluginConfig.objects.get(id=content[0]["id"]) pc2 = PluginConfig.objects.get(id=content[1]["id"]) @@ -1144,7 +1144,7 @@ def test_create(self): response = self.client.post(uri, payload, format="json") self.assertEqual(response.status_code, 201) content = response.json() - self.assertEqual(content[0]["value"], "new_org_supersecret_admin") + self.assertEqual(PluginConfig._decrypt_value(content[0]["value"]), "new_org_supersecret_admin") self.assertEqual(content[0]["owner"], self.admin.username) pc = PluginConfig.objects.get(id=content[0]["id"]) self.assertTrue(pc.for_organization) @@ -1162,7 +1162,7 @@ def test_create(self): response = self.client.post(uri, payload, format="json") self.assertEqual(response.status_code, 201) content = response.json() - self.assertEqual(content[0]["value"], "new_supersecret_admin") + self.assertEqual(PluginConfig._decrypt_value(content[0]["value"]), "new_supersecret_admin") self.assertEqual(content[0]["owner"], self.admin.username) pc1 = PluginConfig.objects.get(id=content[0]["id"]) self.assertFalse(pc1.for_organization) @@ -1197,7 +1197,7 @@ def test_create(self): response = self.client.post(uri, payload, format="json") self.assertEqual(response.status_code, 201) content = response.json() - self.assertEqual(content[0]["value"], "new_supersecret_user_only") + self.assertEqual(PluginConfig._decrypt_value(content[0]["value"]), "new_supersecret_user_only") self.assertEqual(content[0]["owner"], self.user.username) pc = PluginConfig.objects.get(id=content[0]["id"]) self.assertFalse(pc.for_organization) @@ -1244,7 +1244,7 @@ def test_create(self): response = self.client.post(uri, payload, format="json") self.assertEqual(response.status_code, 201) content = response.json() - self.assertEqual(content[0]["value"], "new_user_secret") + self.assertEqual(PluginConfig._decrypt_value(content[0]["value"]), "new_user_secret") self.assertEqual(content[0]["owner"], self.user.username) pc1 = PluginConfig.objects.get(id=content[0]["id"]) self.assertFalse(pc1.for_organization) @@ -1252,7 +1252,7 @@ def test_create(self): pc.delete() param.delete() - PluginConfig.objects.filter(value__startswith="supersecret").delete() + PluginConfig.objects.filter(value__startswith="gAAAAA").delete() org.delete() def test_delete(self): From 20bf725513d59c86d4ea5eb82fd232fc77e5dd73 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 24 Mar 2026 03:57:00 +0530 Subject: [PATCH 5/5] fix: decrypt encrypted URL values in health_check to prevent NotImplementedError --- api_app/classes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api_app/classes.py b/api_app/classes.py index 3a4cf7cfab..f24b428bf2 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -349,6 +349,14 @@ def _get_health_check_url(self, user: User = None) -> typing.Optional[str]: if not param.configured or not param.value: continue url = param.value + # Decrypt if the value is Fernet-encrypted (secret parameter) + if isinstance(url, str) and url.startswith("gAAAAA"): + try: + from api_app.models import PluginConfig + + url = PluginConfig._decrypt_value(url) + except Exception: + pass logger.info(f"Url retrieved to verify is {param.name} for {self}") return url if hasattr(self, "url") and self.url: