Remove NB scripts, introduce distributed RunTests (#102)

* all steps and launcher are ready

* introduce dimi, refactor dependencies

* wip introducing dimi

* introduce dimi

* everything except template is ready

* gui for running tests

* remove two-phase and rollback

* run tests button

* almost working

* fix work splitup

* fully working runtests

* report table and filterset

* remove old scripts

* delete scripts migration

* api for running tests

* fix log debug

* netbox version compatibility

* final tests

* disable fail fast

* add system levels

* backports to support 3.7

* stop running tests for 3.6

* page-header -> header

* remove unneeded template parts

* adjust runtests template

.github/workflows/ci.yml

@@ -21,8 +21,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        netbox_version: [v3.6.9, v3.7.8, v4.0.7]
+        netbox_version: [v3.7.8, v4.0.11]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -58,8 +59,9 @@ jobs:
   test_migrations:
       runs-on: ubuntu-latest
       strategy:
+        fail-fast: false
         matrix:
-          netbox_version: [v3.6.9, v3.7.8, v4.0.2]
+          netbox_version: [v3.7.8, v4.0.11]
       steps:
         - name: Checkout
           uses: actions/checkout@v3

development/.env.example

@@ -6,3 +6,4 @@ REDIS_PASSWORD=redis
 SECRET_KEY=SOME_ARBITRARY_LONG_ENOUGH_DJANGO_SECRET_KEY_STRING
 COMPOSE_PROJECT_NAME=validity
 DEBUGWEB=0
+DEBUGWORKER=0

development/Dockerfile

@@ -16,6 +16,7 @@ RUN mkdir -p /opt/netbox \
 
 # Install Validity
 COPY . /plugin/validity
+COPY ./development/start.sh /opt/netbox/netbox/
 RUN pip install --editable /plugin/validity[dev]
 
 WORKDIR /opt/netbox/netbox/

development/docker-compose.yaml

@@ -5,7 +5,9 @@ services:
       dockerfile: ./development/Dockerfile
       args:
         NETBOX_VERSION: ${NETBOX_VERSION}
-    command: sh -c "python manage.py rqworker"
+    command: ./start.sh $DEBUGWORKER manage.py rqworker
+    ports:
+      - "5679:5678"
     depends_on:
       - postgres
       - redis
@@ -24,13 +26,7 @@ services:
 
   netbox:
     <<: *worker
-    command: >
-      bash -c "
-      if [[ $DEBUGWEB == 1 ]]; then
-      python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000;
-      else
-      python manage.py runserver 0.0.0.0:8000;
-      fi"
+    command: ./start.sh $DEBUGWEB manage.py runserver 0.0.0.0:8000
     ports:
       - "8000:8000"
       - "5678:5678"

development/start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+DEBUG=$1
+shift
+
+if [[ $DEBUG == 1 ]]; then
+    echo "!!! DEBUGGING IS ENABLED !!!"
+    python -m debugpy --listen 0.0.0.0:5678 $@
+else
+    python $@
+fi

pyproject.toml

@@ -60,6 +60,7 @@ branch = true
 omit = [
         "validity/tests/*",
         "validity/migrations/*",
+        "validity/dependencies.py",
 ]
 source = ["validity"]
 

requirements/base.txt

@@ -1,4 +1,5 @@
 deepdiff>=6.2.0,<7
+dimi < 2
 django-bootstrap5 >=24.2,<25
 dulwich # Core NetBox "optional" requirement
 jq>=1.4.0,<2

validity/__init__.py

@@ -1,8 +1,7 @@
 import logging
 
-from django.conf import settings as django_settings
+from dimi import Container
 from netbox.settings import VERSION
-from pydantic import BaseModel, Field
 
 from validity.utils.version import NetboxVersion
 
@@ -31,20 +30,12 @@ class NetBoxValidityConfig(PluginConfig):
     netbox_version = NetboxVersion(VERSION)
 
     def ready(self):
-        import validity.data_backends
+        from validity import data_backends, dependencies, signals
 
         return super().ready()
 
 
 config = NetBoxValidityConfig
 
 
-class ValiditySettings(BaseModel):
-    store_last_results: int = Field(default=5, gt=0, lt=1001)
-    store_reports: int = Field(default=5, gt=0, lt=1001)
-    sleep_between_tests: float = 0
-    result_batch_size: int = Field(default=500, ge=1)
-    polling_threads: int = Field(default=500, ge=1)
-
-
-settings = ValiditySettings.model_validate(django_settings.PLUGINS_CONFIG.get("validity", {}))
+di = Container()

validity/api/helpers.py

@@ -5,6 +5,7 @@
 from django.core.exceptions import ValidationError
 from django.db.models import ManyToManyField
 from netbox.api.serializers import WritableNestedSerializer
+from rest_framework.relations import PrimaryKeyRelatedField
 from rest_framework.serializers import JSONField, ModelSerializer
 
 from validity import NetboxVersion
@@ -94,3 +95,13 @@ def validate(self, attrs):
                 ]
                 raise ValidationError({instance.subform_json_field: errors})
         return attrs
+
+
+class PrimaryKeyField(PrimaryKeyRelatedField):
+    """
+    Returns primary key only instead of the whole model instance
+    """
+
+    def to_internal_value(self, data):
+        obj = super().to_internal_value(data)
+        return obj.pk

validity/api/serializers.py

@@ -1,4 +1,6 @@
 from core.api.nested_serializers import NestedDataFileSerializer, NestedDataSourceSerializer
+from core.api.serializers import JobSerializer
+from core.models import DataSource
 from dcim.api.nested_serializers import (
     NestedDeviceSerializer,
     NestedDeviceTypeSerializer,
@@ -7,7 +9,9 @@
     NestedPlatformSerializer,
     NestedSiteSerializer,
 )
-from dcim.models import DeviceType, Location, Manufacturer, Platform, Site
+from dcim.models import Device, DeviceType, Location, Manufacturer, Platform, Site
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
 from extras.api.nested_serializers import NestedTagSerializer
 from extras.models import Tag
 from netbox.api.fields import SerializedPKRelatedField
@@ -18,7 +22,15 @@
 from tenancy.models import Tenant
 
 from validity import config, models
-from .helpers import EncryptedDictField, FieldsMixin, ListQPMixin, SubformValidationMixin, nested_factory
+from validity.choices import ExplanationVerbosityChoices
+from .helpers import (
+    EncryptedDictField,
+    FieldsMixin,
+    ListQPMixin,
+    PrimaryKeyField,
+    SubformValidationMixin,
+    nested_factory,
+)
 
 
 class ComplianceSelectorSerializer(NetBoxModelSerializer):
@@ -370,3 +382,30 @@ def to_representation(self, instance):
         if name_filter := self.get_list_param("name"):
             instance = [item for item in instance if item.name in set(name_filter)]
         return super().to_representation(instance)
+
+
+class RunTestsSerializer(serializers.Serializer):
+    sync_datasources = serializers.BooleanField(required=False)
+    selectors = PrimaryKeyField(
+        many=True,
+        required=False,
+        queryset=models.ComplianceSelector.objects.all(),
+    )
+    devices = PrimaryKeyField(many=True, required=False, queryset=Device.objects.all())
+    test_tags = PrimaryKeyField(many=True, required=False, queryset=Tag.objects.all())
+    explanation_verbosity = serializers.ChoiceField(
+        choices=ExplanationVerbosityChoices.choices, required=False, default=ExplanationVerbosityChoices.maximum
+    )
+    overriding_datasource = PrimaryKeyField(required=False, queryset=DataSource.objects.all())
+    workers_num = serializers.IntegerField(min_value=1, default=1)
+    schedule_at = serializers.DateTimeField(required=False, allow_null=True)
+    schedule_interval = serializers.IntegerField(required=False, allow_null=True)
+
+    def validate_schedule_at(self, value):
+        if value and value < timezone.now():
+            raise serializers.ValidationError(_("Scheduled time must be in the future."))
+        return value
+
+
+class ScriptResultSerializer(serializers.Serializer):
+    result = JobSerializer(read_only=True)

validity/api/views.py

@@ -1,17 +1,39 @@
-from drf_spectacular.utils import OpenApiParameter, extend_schema
-from netbox.api.viewsets import NetBoxModelViewSet
+from http import HTTPStatus
+from typing import Annotated, Any
+
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
+from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet
+from rest_framework.decorators import action
 from rest_framework.exceptions import NotFound
 from rest_framework.generics import ListAPIView
 from rest_framework.response import Response
+from rest_framework.serializers import Serializer
 from rest_framework.views import APIView
 
-from validity import filtersets, models
+from validity import di, filtersets, models
 from validity.choices import SeverityChoices
+from validity.scripts import Launcher, RunTestsParams, ScriptParams
 from . import serializers
 
 
-class ReadOnlyNetboxViewSet(NetBoxModelViewSet):
-    http_method_names = ["get", "head", "options", "trace"]
+class RunMixin:
+    run_serializer_class: type[Serializer]
+    params_class: type[ScriptParams]
+    launcher: Launcher
+
+    def get_params(self, serializer, request):
+        return self.params_class(**serializer.validated_data, request=request)
+
+    def get_result_data(self, job, request):
+        serializer = serializers.ScriptResultSerializer({"result": job}, context={"request": request})
+        return serializer.data
+
+    def run(self, request):
+        serializer = self.run_serializer_class(data=request.data)
+        if not serializer.is_valid():
+            return Response(status=HTTPStatus.BAD_REQUEST, data=serializer.errors)
+        job = self.launcher(self.get_params(serializer, request))
+        return Response(self.get_result_data(job, request))
 
 
 class ComplianceSelectorViewSet(NetBoxModelViewSet):
@@ -29,15 +51,27 @@ class ComplianceSelectorViewSet(NetBoxModelViewSet):
     filterset_class = filtersets.ComplianceSelectorFilterSet
 
 
-class ComplianceTestViewSet(NetBoxModelViewSet):
+@extend_schema_view(run=extend_schema(request=serializers.RunTestsSerializer))
+class ComplianceTestViewSet(RunMixin, NetBoxModelViewSet):
     queryset = models.ComplianceTest.objects.select_related("data_source", "data_file").prefetch_related(
         "selectors", "tags"
     )
     serializer_class = serializers.ComplianceTestSerializer
     filterset_class = filtersets.ComplianceTestFilterSet
+    run_serializer_class = serializers.RunTestsSerializer
+    params_class = RunTestsParams
+
+    @di.inject
+    def __init__(self, launcher: Annotated[Launcher, "runtests_launcher"], **kwargs: Any) -> None:
+        self.launcher = launcher
+        super().__init__(**kwargs)
+
+    @action(detail=False, methods=["post"], url_path="run")
+    def run(self, request):
+        return super().run(request)
 
 
-class ComplianceTestResultViewSet(ReadOnlyNetboxViewSet):
+class ComplianceTestResultViewSet(NetBoxReadOnlyModelViewSet):
     queryset = models.ComplianceTestResult.objects.select_related("device", "test", "report")
     serializer_class = serializers.ComplianceTestResultSerializer
     filterset_class = filtersets.ComplianceTestResultFilterSet
@@ -55,10 +89,10 @@ class NameSetViewSet(NetBoxModelViewSet):
     filterset_class = filtersets.NameSetFilterSet
 
 
-class ComplianceReportViewSet(NetBoxModelViewSet):
+class ComplianceReportViewSet(NetBoxReadOnlyModelViewSet):
     queryset = models.ComplianceReport.objects.annotate_result_stats().count_devices_and_tests()
     serializer_class = serializers.ComplianceReportSerializer
-    http_method_names = ["get", "head", "options", "trace", "delete"]
+    filterset_class = filtersets.ComplianceReportFilterSet
 
 
 class PollerViewSet(NetBoxModelViewSet):

validity/dependencies.py

@@ -0,0 +1,64 @@
+from typing import Annotated
+
+import django_rq
+from dimi.scopes import Singleton
+from django.conf import LazySettings, settings
+from utilities.rqworker import get_workers_for_queue
+
+from validity import di
+from validity.choices import ConnectionTypeChoices
+from validity.pollers import NetmikoPoller, RequestsPoller, ScrapliNetconfPoller
+from validity.settings import ValiditySettings
+from validity.utils.misc import null_request
+
+
+@di.dependency
+def django_settings():
+    return settings
+
+
+@di.dependency(scope=Singleton)
+def validity_settings(django_settings: Annotated[LazySettings, django_settings]):
+    return ValiditySettings.model_validate(django_settings.PLUGINS_CONFIG.get("validity", {}))
+
+
+@di.dependency(scope=Singleton)
+def poller_map():
+    return {
+        ConnectionTypeChoices.netmiko: NetmikoPoller,
+        ConnectionTypeChoices.requests: RequestsPoller,
+        ConnectionTypeChoices.scrapli_netconf: ScrapliNetconfPoller,
+    }
+
+
+from validity.scripts import ApplyWorker, CombineWorker, Launcher, SplitWorker, Task  # noqa
+
+
+@di.dependency
+def runtests_worker_count(vsettings: Annotated[ValiditySettings, validity_settings]) -> int:
+    return get_workers_for_queue(vsettings.runtests_queue)
+
+
+@di.dependency(scope=Singleton)
+def runtests_launcher(
+    vsettings: Annotated[ValiditySettings, validity_settings],
+    split_worker: Annotated[SplitWorker, ...],
+    apply_worker: Annotated[ApplyWorker, ...],
+    combine_worker: Annotated[CombineWorker, ...],
+):
+    from validity.models import ComplianceReport
+
+    return Launcher(
+        job_name="RunTests",
+        job_object_factory=null_request()(ComplianceReport.objects.create),
+        rq_queue=django_rq.get_queue(vsettings.runtests_queue),
+        tasks=[
+            Task(split_worker, job_timeout=vsettings.script_timeouts.runtests_split),
+            Task(
+                apply_worker,
+                job_timeout=vsettings.script_timeouts.runtests_apply,
+                multi_workers=True,
+            ),
+            Task(combine_worker, job_timeout=vsettings.script_timeouts.runtests_combine),
+        ],
+    )

validity/filtersets.py

@@ -2,12 +2,14 @@
 from functools import reduce
 from typing import Sequence
 
+from core.choices import JobStatusChoices
+from core.models import Job
 from dcim.filtersets import DeviceFilterSet
 from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, Platform, Site
 from django.db.models import Q
 from django_filters import BooleanFilter, ChoiceFilter, ModelMultipleChoiceFilter
 from extras.models import Tag
-from netbox.filtersets import NetBoxModelFilterSet
+from netbox.filtersets import ChangeLoggedModelFilterSet, NetBoxModelFilterSet
 from tenancy.models import Tenant
 
 from validity import models
@@ -112,6 +114,15 @@ class DeviceReportFilterSet(DeviceFilterSet):
     compliance_passed = BooleanFilter()
 
 
+class ComplianceReportFilterSet(ChangeLoggedModelFilterSet):
+    job_status = ChoiceFilter(field_name="jobs__status", choices=JobStatusChoices)
+    job_id = ModelMultipleChoiceFilter(field_name="jobs", queryset=Job.objects.all())
+
+    class Meta:
+        model = models.ComplianceReport
+        fields = ("id", "job_id", "job_status", "created")
+
+
 class PollerFilterSet(SearchMixin, NetBoxModelFilterSet):
     class Meta:
         model = models.Poller