Add operator test cases

Update CI to run tests

Author: meffmadd

.github/workflows/ci.yml

@@ -31,10 +31,50 @@ jobs:
       - id: outputStep
         run: echo "changeDirs=${{ steps.changeDirsStep.outputs.all_changed_files }}" >> $GITHUB_OUTPUT
 
+
+  server-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install -r testing.requirements.txt
+          pip install -r srv/requirements.txt
+
+      - name: Run tests
+        run: pytest tests/server_tests
+
+  operator-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.11'
+
+      - name: Setup k3s test cluster
+        uses: nolar/setup-k3d-k3s@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          pip install -r testing.requirements.txt
+          pip install -r opr/requirements.txt
+
+      - name: Run tests
+        run: pytest tests/operator_tests
+
   docker:
     runs-on: ubuntu-latest
     needs:
       - changed-dirs
+      - server-tests
+      - operator-tests
     if: contains(needs.changed-dirs.outputs.changeDirs, 'opr') || contains(needs.changed-dirs.outputs.changeDirs, 'srv')
     strategy:
       matrix:

tests/operator_tests/__init__.py

@@ -0,0 +1,86 @@
+import os
+import uuid
+import pytest
+import kubernetes
+from kubernetes.client import ApiClient, CoreV1Api, V1Namespace, V1ObjectMeta, AppsV1Api, CustomObjectsApi, ApiextensionsV1Api
+
+
+def delete_all(namespace, list_func, delete_func, patch_func):
+    resources = list_func(namespace=namespace)
+    try:
+        resources_list = resources["items"]
+    except TypeError:
+        resources_list = resources.items
+    for resource in resources_list:
+        # Operator is not running in fixtures, so we need a force-delete (or this patch).
+        patch_body = {
+            "metadata": {
+                "finalizers": []
+            }
+        }
+
+        try:
+            name = resource.metadata.name
+        except AttributeError:
+            name = resource["metadata"]["name"]
+
+        try:
+            patch_func(name=name, namespace=namespace, body=patch_body)
+        except kubernetes.client.exceptions.ApiException as e:
+            if e.status == 404:
+                pass
+
+        try:
+            delete_func(name=name, namespace=namespace)
+        except kubernetes.client.exceptions.ApiException as e:
+            if e.status == 404:
+                pass
+
+
+def delete_all_custom_objects(crd_api, namespace, plural):
+    def list_cr(namespace):
+        return crd_api.list_namespaced_custom_object("datalab.tuwien.ac.at", "v1", namespace, plural)
+
+    def delete_cr(name, namespace):
+        return crd_api.delete_namespaced_custom_object("datalab.tuwien.ac.at", "v1", namespace, plural, name=name)
+
+    def patch_cr(name, namespace, body):
+        return crd_api.patch_namespaced_custom_object("datalab.tuwien.ac.at", "v1", namespace, plural, name=name, body=body)
+
+    delete_all(namespace, list_cr, delete_cr, patch_cr)
+
+
+@pytest.fixture(scope="function")
+def random_namespace():
+    client = ApiClient(configuration=kubernetes.config.load_kube_config())
+    api = CoreV1Api(api_client=client)
+
+    namespace = f'test-namespace-{uuid.uuid4().hex[:10]}'
+    try:
+        body = V1Namespace(metadata=V1ObjectMeta(name=namespace))
+        api.create_namespace(body=body)
+        yield namespace
+    finally:
+        apps_api = AppsV1Api(api_client=client)
+        crd_api = CustomObjectsApi(api_client=client)
+        extensions_api = ApiextensionsV1Api(api_client=client)
+
+        delete_all(namespace, api.list_namespaced_pod, api.delete_namespaced_pod, api.patch_namespaced_pod)
+        delete_all(namespace, api.list_namespaced_config_map, api.delete_namespaced_config_map, api.patch_namespaced_config_map)
+        delete_all(namespace, api.list_namespaced_service, api.delete_namespaced_service, api.patch_namespaced_service)
+        delete_all(namespace, apps_api.list_namespaced_deployment, apps_api.delete_namespaced_deployment, apps_api.patch_namespaced_deployment)
+
+        delete_all_custom_objects(crd_api, namespace, "keyvaluepairs")
+        delete_all_custom_objects(crd_api, namespace, "configservers")
+
+        config_server_crds = ["configservers.datalab.tuwien.ac.at", "keyvaluepairs.datalab.tuwien.ac.at"]
+        for name in config_server_crds:
+            extensions_api.delete_custom_resource_definition(name=name)
+
+        api.delete_namespace(name=namespace)
+        client.close()
+
+
+@pytest.fixture(scope="session")
+def operator_file():
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), '../../opr/operator.py'))

tests/operator_tests/test_config_server.py

@@ -0,0 +1,95 @@
+import kubernetes
+from kubernetes import watch
+from kubernetes.client import ApiClient, CustomObjectsApi, CoreV1Api, V1ConfigMap, V1Volume, V1Pod
+from kopf.testing import KopfRunner
+
+from . import random_namespace, operator_file
+from .test_crd import create_crd
+
+
+def create_config_server(client: ApiClient, namespace: str):
+    custom_objects_api = CustomObjectsApi(client)
+    body = {
+        "apiVersion": "datalab.tuwien.ac.at/v1",
+        "kind": "ConfigServer",
+        "metadata": {
+            "name": "test-config-server",
+            "namespace": namespace
+        },
+        "spec": {
+            "image": "ghcr.io/tu-wien-datalab/config-server:main",
+            "imagePullPolicy": "IfNotPresent",
+            "containerPort": 80,
+            "configMountPath": "/var/lib/config-server"
+        }
+
+    }
+    custom_objects_api.create_namespaced_custom_object("datalab.tuwien.ac.at", "v1", namespace, "configservers", body)
+
+
+def test_config_server_custom_resource(random_namespace, operator_file):
+    client = ApiClient(configuration=kubernetes.config.load_kube_config())
+    core_api = CoreV1Api(client)
+    custom_objects_api = CustomObjectsApi(client)
+
+    create_crd(client)
+
+    with KopfRunner(['run', '-A', '--verbose', operator_file]) as runner:
+        create_config_server(client, random_namespace)
+
+        config_server_watch = watch.Watch()
+        entered = False
+        for event in config_server_watch.stream(custom_objects_api.list_namespaced_custom_object, "datalab.tuwien.ac.at", "v1", random_namespace, "configservers"):
+            assert event['type'] == "ADDED"
+            obj = event['object']  # object is one of type return_type
+
+            entered = True
+            assert obj["metadata"]["name"] == "test-config-server"
+            config_server_watch.stop()
+        assert entered
+
+        pod_watch = watch.Watch()
+        entered = False
+
+        for event in pod_watch.stream(core_api.list_namespaced_pod, random_namespace, timeout_seconds=10):
+            assert event['type'] == "ADDED"
+
+            obj = event['object']  # object is one of type return_type
+            assert isinstance(obj, V1Pod)
+
+            entered = True
+
+            pod = obj
+            assert pod.metadata.name.startswith("test-config-server")
+            assert pod.status.phase in ["Running", "Pending"]
+            volumes: list[V1Volume] = pod.spec.volumes
+            assert any([v.config_map is not None for v in volumes])
+            config_volumes = list(filter(lambda v: v.config_map is not None, volumes))
+            assert len(config_volumes) == 1
+            assert config_volumes[0].name == "config"
+            config_map = config_volumes[0].config_map
+            assert config_map.name == "test-config-server-values"
+
+            pod_watch.stop()
+        assert entered
+
+        config_map_watch = watch.Watch()
+        entered = False
+        for event in config_map_watch.stream(core_api.list_namespaced_config_map, random_namespace, timeout_seconds=10):
+            assert event['type'] == "ADDED"
+
+            obj = event['object']  # object is one of type return_type
+            assert isinstance(obj, V1ConfigMap)
+
+            config_map: V1ConfigMap = obj
+            if config_map.metadata.name != "test-config-server-values":
+                continue
+
+            entered = True
+            assert config_map.data is None
+
+            config_map_watch.stop()
+        assert entered
+
+    assert runner.exit_code == 0
+    assert runner.exception is None

tests/operator_tests/test_crd.py

@@ -0,0 +1,30 @@
+import os
+
+import pytest
+import kubernetes
+from kubernetes.client import ApiClient, ApiextensionsV1Api, V1CustomResourceDefinition
+from kubernetes import utils
+
+
+def create_crd(client: ApiClient):
+    yaml_file = os.path.join(os.path.dirname(__file__), '../../opr/crd.yaml')
+    assert os.path.exists(yaml_file)
+    utils.create_from_yaml(client, yaml_file)
+
+
+def test_create_crd():
+    client = ApiClient(configuration=kubernetes.config.load_kube_config())
+    extensions_api = ApiextensionsV1Api(api_client=client)
+
+    create_crd(client)
+
+    config_server_crds = ["configservers.datalab.tuwien.ac.at", "keyvaluepairs.datalab.tuwien.ac.at"]
+    list_crds = lambda namespace: extensions_api.list_custom_resource_definition()
+    try:
+        crds: list[V1CustomResourceDefinition] = list_crds(None).items
+        names = [c.metadata.name for c in crds]
+        assert len(names) > 0
+        assert set(names).issuperset(config_server_crds)
+    finally:
+        for name in config_server_crds:
+            extensions_api.delete_custom_resource_definition(name=name)