Fix and improve for local deployment

config/pipeline.yaml

@@ -6,7 +6,7 @@
 api:
 
   docker-host:
-    url: http://172.17.0.1:8001
+    url: http://host.docker.internal:8001
 
   staging:
     url: https://staging.kernelci.org:9000
@@ -22,9 +22,9 @@ storage:
 
   docker-host:
     storage_type: ssh
-    host: 172.17.0.1
+    host: host.docker.internal
     port: 8022
-    base_url: http://172.17.0.1:8002/
+    base_url: http://host.docker.internal:8002/
 
   production-azure: &azure-files
     storage_type: azure

doc/connecting-lab.md

@@ -75,7 +75,7 @@ runtime_token="N0tAS3creTT0k3n"
 ### `docker-compose` file
 
 We are running all the pipeline services as docker containers.
-You need to provide lab name to `--runtimes` argument to the [`scheduler-lava`](https://github.com/kernelci/kernelci-pipeline/blob/main/docker-compose.yaml#L80)
+You need to provide lab name to `--runtimes` argument to the [`scheduler-lava`](https://github.com/kernelci/kernelci-pipeline/blob/main/docker-compose.yaml#L93)
 service in the `docker-compose.yml` file to enable the lab.
 For example, the following configuration adds the `lava-broonie` lab along with other labs:
 
@@ -84,13 +84,17 @@ scheduler-lava:
     <<: *scheduler
     container_name: 'kernelci-pipeline-scheduler-lava'
     command:
-      - './pipeline/scheduler.py'
+      - './src/scheduler.py'
+      - '--yaml-config=/home/kernelci/config'
       - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
       - 'loop'
+      - '--name=scheduler_lava'
       - '--runtimes'
       - 'lava-collabora'
       - 'lava-collabora-staging'
       - 'lava-broonie'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 ```
 
 ### Jobs and devices specific to the lab

docker-compose-production.yaml

@@ -3,8 +3,6 @@
 # Copyright (C) 2023 Collabora Limited
 # Author: Guillaume Tucker <[email protected]>
 
-version: '3'
-
 services:
 
   lava-callback:

docker-compose.yaml

@@ -4,7 +4,11 @@
 # Author: Guillaume Tucker <[email protected]>
 # Author: Jeny Sadadia <[email protected]>
 
-version: '3'
+x-config-arg: &config-arg
+  --yaml-config=/home/kernelci/config
+
+x-setting-arg: &setting-arg
+  --settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}
 
 services:
 
@@ -15,38 +19,33 @@ services:
     stop_signal: 'SIGINT'
     command:
       - './src/monitor.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'run'
-    volumes: &base-volumes
-#      - './src:/home/kernelci/pipeline'
+    volumes:
       - './config:/home/kernelci/config'
       - './logs:/home/kernelci/logs'
     extra_hosts:
       - "host.docker.internal:host-gateway"
     restart: on-failure
 
   scheduler: &scheduler
+    <<: *base-service
     container_name: 'kernelci-pipeline-scheduler'
-    image: 'kernelci/staging-kernelci:pipeline'
-    env_file: ['.env']
-    stop_signal: 'SIGINT'
     command:
       - './src/scheduler.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'loop'
       - '--runtimes=shell'
       - '--name=scheduler'
     volumes:
-#      - './src:/home/kernelci/pipeline'
       - './config:/home/kernelci/config'
       - './data/output:/home/kernelci/data/output'
       - './data/k8s-credentials/.kube:/home/kernelci/.kube'
       - './data/k8s-credentials/.config/gcloud:/home/kernelci/.config/gcloud'
       - './data/k8s-credentials/.azure:/home/kernelci/.azure'
       - './logs:/home/kernelci/logs'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    restart: on-failure
 
   scheduler-docker:
     <<: *scheduler
@@ -55,26 +54,25 @@ services:
     #working_dir: /home/kernelci
     command:
       - './src/scheduler.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'loop'
       - '--runtimes=docker'
       - '--name=scheduler_docker'
     volumes:
-#      - './src:/home/kernelci/pipeline'
       - './config:/home/kernelci/config'
       - './data/output:/home/kernelci/data/output'
       - './.docker-env:/home/kernelci/.docker-env'
       - '/var/run/docker.sock:/var/run/docker.sock'  # Docker-in-Docker
       - './logs:/home/kernelci/logs'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   scheduler-lava:
     <<: *scheduler
     container_name: 'kernelci-pipeline-scheduler-lava'
     command:
       - './src/scheduler.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'loop'
       - '--name=scheduler_lava'
       - '--runtimes'
@@ -87,105 +85,128 @@ services:
       - 'lava-cip'
       - 'lava-pengutronix'
       - 'lava-kontron'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   scheduler-k8s:
     <<: *scheduler
     container_name: 'kernelci-pipeline-scheduler-k8s'
-    image: 'kernelci/staging-kernelci:pipeline'
     command:
       - './src/scheduler.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'loop'
       - '--name=scheduler_k8s'
       - '--runtimes'
       - 'k8s-gke-eu-west4'
       - 'k8s-all'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   tarball:
     <<: *base-service
     container_name: 'kernelci-pipeline-tarball'
     command:
       - './src/tarball.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'run'
       - '--name=tarball'
     volumes:
-#      - './src:/home/kernelci/pipeline'
       - './config:/home/kernelci/config'
       - './data/ssh:/home/kernelci/data/ssh'
       - './data/src:/home/kernelci/data/src'
       - './data/output:/home/kernelci/data/output'
       - './logs:/home/kernelci/logs'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   trigger:
     <<: *base-service
     container_name: 'kernelci-pipeline-trigger'
     command:
       - './src/trigger.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'run'
       - '--trees=kernelci,netdev-testing'
       - '--name=trigger'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   timeout:
     <<: *base-service
     container_name: 'kernelci-pipeline-timeout'
     command:
-      - '/usr/bin/env'
-      - 'python3'
-      - '/home/kernelci/pipeline/src/timeout.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - './src/timeout.py'
+      - *config-arg
+      - *setting-arg
       - 'run'
       - '--mode=timeout'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   timeout-closing:
     <<: *base-service
     container_name: 'kernelci-pipeline-closing'
     command:
-      - '/usr/bin/env'
-      - 'python3'
-      - '/home/kernelci/pipeline/src/timeout.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - './src/timeout.py'
+      - *config-arg
+      - *setting-arg
       - 'run'
       - '--mode=closing'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   timeout-holdoff:
     <<: *base-service
     container_name: 'kernelci-pipeline-holdoff'
     command:
-      - '/usr/bin/env'
-      - 'python3'
-      - '/home/kernelci/pipeline/src/timeout.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - './src/timeout.py'
+      - *config-arg
+      - *setting-arg
       - 'run'
       - '--mode=holdoff'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
 
   patchset:
     <<: *base-service
     container_name: 'kernelci-pipeline-patchset'
     command:
       - './src/patchset.py'
-      - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}'
+      - *config-arg
+      - *setting-arg
       - 'run'
       - '--name=patchset'
     volumes:
-#      - './src:/home/kernelci/pipeline'
       - './config:/home/kernelci/config'
       - './data/ssh:/home/kernelci/data/ssh'
       - './data/src:/home/kernelci/data/src'
       - './data/output:/home/kernelci/data/output'
       - './logs:/home/kernelci/logs'
+
+  lava-callback:
+    <<: *base-service
+    container_name: 'kernelci-pipeline-lava-callback'
+    image: 'kernelci/staging-kernelci:lava-callback'
+    ports:
+      - '${LAVA_CALLBACK_PORT:-8100}:8000'
+    # With Werkzeug development server, not suitable for a public instance
+    command:
+      - './src/lava_callback.py'
+      - *config-arg
+      - *setting-arg
+    volumes:
+      - './config:/home/kernelci/config'
+      - './data/ssh:/home/kernelci/data/ssh'
+
+  kcidb:
+    <<: *base-service
+    container_name: 'kernelci-pipeline-kcidb'
+    command:
+      - './src/send_kcidb.py'
+      - *config-arg
+      - *setting-arg
+      - 'run'
+      - '--name=pipeline_kcidb'
+    volumes:
+      - './config:/home/kernelci/config'
+      - './data/kcidb:/home/kernelci/data/kcidb'
+      - './logs:/home/kernelci/logs'
+
+  postgres:
+    container_name: 'kernelci-pipeline-postgres-proxy'
+    image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy:latest'
+    command:
+      - 'kernelci-production:us-central1:postgresql2'
+      - '-c/home/kernelci/data/kcidb/kernelci-production-ci-kernelci.json'
+      - '-a0.0.0.0'
+    volumes:
+      - './data/kcidb:/home/kernelci/data/kcidb'