Remove NB scripts, introduce distributed RunTests (#102)

* all steps and launcher are ready

* introduce dimi, refactor dependencies

* wip introducing dimi

* introduce dimi

* everything except template is ready

* gui for running tests

* remove two-phase and rollback

* run tests button

* almost working

* fix work splitup

* fully working runtests

* report table and filterset

* remove old scripts

* delete scripts migration

* api for running tests

* fix log debug

* netbox version compatibility

* final tests

* disable fail fast

* add system levels

* backports to support 3.7

* stop running tests for 3.6

* page-header -> header

* remove unneeded template parts

* adjust runtests template

Author: amyasnikov

.github/workflows/ci.yml

@@ -21,8 +21,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        netbox_version: [v3.6.9, v3.7.8, v4.0.7]
+        netbox_version: [v3.7.8, v4.0.11]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -58,8 +59,9 @@ jobs:
   test_migrations:
       runs-on: ubuntu-latest
       strategy:
+        fail-fast: false
         matrix:
-          netbox_version: [v3.6.9, v3.7.8, v4.0.2]
+          netbox_version: [v3.7.8, v4.0.11]
       steps:
         - name: Checkout
           uses: actions/checkout@v3

development/.env.example

@@ -6,3 +6,4 @@ REDIS_PASSWORD=redis
 SECRET_KEY=SOME_ARBITRARY_LONG_ENOUGH_DJANGO_SECRET_KEY_STRING
 COMPOSE_PROJECT_NAME=validity
 DEBUGWEB=0
+DEBUGWORKER=0

development/Dockerfile

@@ -16,6 +16,7 @@ RUN mkdir -p /opt/netbox \
 
 # Install Validity
 COPY . /plugin/validity
+COPY ./development/start.sh /opt/netbox/netbox/
 RUN pip install --editable /plugin/validity[dev]
 
 WORKDIR /opt/netbox/netbox/

development/docker-compose.yaml

@@ -5,7 +5,9 @@ services:
       dockerfile: ./development/Dockerfile
       args:
         NETBOX_VERSION: ${NETBOX_VERSION}
-    command: sh -c "python manage.py rqworker"
+    command: ./start.sh $DEBUGWORKER manage.py rqworker
+    ports:
+      - "5679:5678"
     depends_on:
       - postgres
       - redis
@@ -24,13 +26,7 @@ services:
 
   netbox:
     <<: *worker
-    command: >
-      bash -c "
-      if [[ $DEBUGWEB == 1 ]]; then
-      python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000;
-      else
-      python manage.py runserver 0.0.0.0:8000;
-      fi"
+    command: ./start.sh $DEBUGWEB manage.py runserver 0.0.0.0:8000
     ports:
       - "8000:8000"
       - "5678:5678"

development/start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+DEBUG=$1
+shift
+
+if [[ $DEBUG == 1 ]]; then
+    echo "!!! DEBUGGING IS ENABLED !!!"
+    python -m debugpy --listen 0.0.0.0:5678 $@
+else
+    python $@
+fi

pyproject.toml

@@ -60,6 +60,7 @@ branch = true
 omit = [
         "validity/tests/*",
         "validity/migrations/*",
+        "validity/dependencies.py",
 ]
 source = ["validity"]
 

requirements/base.txt

@@ -1,4 +1,5 @@
 deepdiff>=6.2.0,<7
+dimi < 2
 django-bootstrap5 >=24.2,<25
 dulwich # Core NetBox "optional" requirement
 jq>=1.4.0,<2

validity/__init__.py

@@ -1,8 +1,7 @@
 import logging
 
-from django.conf import settings as django_settings
+from dimi import Container
 from netbox.settings import VERSION
-from pydantic import BaseModel, Field
 
 from validity.utils.version import NetboxVersion
 
@@ -31,20 +30,12 @@ class NetBoxValidityConfig(PluginConfig):
     netbox_version = NetboxVersion(VERSION)
 
     def ready(self):
-        import validity.data_backends
+        from validity import data_backends, dependencies, signals
 
         return super().ready()
 
 
 config = NetBoxValidityConfig
 
 
-class ValiditySettings(BaseModel):
-    store_last_results: int = Field(default=5, gt=0, lt=1001)
-    store_reports: int = Field(default=5, gt=0, lt=1001)
-    sleep_between_tests: float = 0
-    result_batch_size: int = Field(default=500, ge=1)
-    polling_threads: int = Field(default=500, ge=1)
-
-
-settings = ValiditySettings.model_validate(django_settings.PLUGINS_CONFIG.get("validity", {}))
+di = Container()

validity/api/helpers.py

@@ -5,6 +5,7 @@
 from django.core.exceptions import ValidationError
 from django.db.models import ManyToManyField
 from netbox.api.serializers import WritableNestedSerializer
+from rest_framework.relations import PrimaryKeyRelatedField
 from rest_framework.serializers import JSONField, ModelSerializer
 
 from validity import NetboxVersion
@@ -94,3 +95,13 @@ def validate(self, attrs):
                 ]
                 raise ValidationError({instance.subform_json_field: errors})
         return attrs
+
+
+class PrimaryKeyField(PrimaryKeyRelatedField):
+    """
+    Returns primary key only instead of the whole model instance
+    """
+
+    def to_internal_value(self, data):
+        obj = super().to_internal_value(data)
+        return obj.pk

validity/api/serializers.py

@@ -1,4 +1,6 @@
 from core.api.nested_serializers import NestedDataFileSerializer, NestedDataSourceSerializer
+from core.api.serializers import JobSerializer
+from core.models import DataSource
 from dcim.api.nested_serializers import (
     NestedDeviceSerializer,
     NestedDeviceTypeSerializer,
@@ -7,7 +9,9 @@
     NestedPlatformSerializer,
     NestedSiteSerializer,
 )
-from dcim.models import DeviceType, Location, Manufacturer, Platform, Site
+from dcim.models import Device, DeviceType, Location, Manufacturer, Platform, Site
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
 from extras.api.nested_serializers import NestedTagSerializer
 from extras.models import Tag
 from netbox.api.fields import SerializedPKRelatedField
@@ -18,7 +22,15 @@
 from tenancy.models import Tenant
 
 from validity import config, models
-from .helpers import EncryptedDictField, FieldsMixin, ListQPMixin, SubformValidationMixin, nested_factory
+from validity.choices import ExplanationVerbosityChoices
+from .helpers import (
+    EncryptedDictField,
+    FieldsMixin,
+    ListQPMixin,
+    PrimaryKeyField,
+    SubformValidationMixin,
+    nested_factory,
+)
 
 
 class ComplianceSelectorSerializer(NetBoxModelSerializer):
@@ -370,3 +382,30 @@ def to_representation(self, instance):
         if name_filter := self.get_list_param("name"):
             instance = [item for item in instance if item.name in set(name_filter)]
         return super().to_representation(instance)
+
+
+class RunTestsSerializer(serializers.Serializer):
+    sync_datasources = serializers.BooleanField(required=False)
+    selectors = PrimaryKeyField(
+        many=True,
+        required=False,
+        queryset=models.ComplianceSelector.objects.all(),
+    )
+    devices = PrimaryKeyField(many=True, required=False, queryset=Device.objects.all())
+    test_tags = PrimaryKeyField(many=True, required=False, queryset=Tag.objects.all())
+    explanation_verbosity = serializers.ChoiceField(
+        choices=ExplanationVerbosityChoices.choices, required=False, default=ExplanationVerbosityChoices.maximum
+    )
+    overriding_datasource = PrimaryKeyField(required=False, queryset=DataSource.objects.all())
+    workers_num = serializers.IntegerField(min_value=1, default=1)
+    schedule_at = serializers.DateTimeField(required=False, allow_null=True)
+    schedule_interval = serializers.IntegerField(required=False, allow_null=True)
+
+    def validate_schedule_at(self, value):
+        if value and value < timezone.now():
+            raise serializers.ValidationError(_("Scheduled time must be in the future."))
+        return value
+
+
+class ScriptResultSerializer(serializers.Serializer):
+    result = JobSerializer(read_only=True)