diff --git a/config/pipeline.yaml b/config/pipeline.yaml index 16dd246ad..65b92b0da 100644 --- a/config/pipeline.yaml +++ b/config/pipeline.yaml @@ -6,7 +6,7 @@ api: docker-host: - url: http://172.17.0.1:8001 + url: http://host.docker.internal:8001 staging: url: https://staging.kernelci.org:9000 @@ -22,9 +22,9 @@ storage: docker-host: storage_type: ssh - host: 172.17.0.1 + host: host.docker.internal port: 8022 - base_url: http://172.17.0.1:8002/ + base_url: http://host.docker.internal:8002/ production-azure: &azure-files storage_type: azure diff --git a/doc/connecting-lab.md b/doc/connecting-lab.md index 27dad46ac..c6ce09dfc 100644 --- a/doc/connecting-lab.md +++ b/doc/connecting-lab.md @@ -75,7 +75,7 @@ runtime_token="N0tAS3creTT0k3n" ### `docker-compose` file We are running all the pipeline services as docker containers. -You need to provide lab name to `--runtimes` argument to the [`scheduler-lava`](https://github.com/kernelci/kernelci-pipeline/blob/main/docker-compose.yaml#L80) +You need to provide lab name to `--runtimes` argument to the [`scheduler-lava`](https://github.com/kernelci/kernelci-pipeline/blob/main/docker-compose.yaml#L93) service in the `docker-compose.yml` file to enable the lab. For example, the following configuration adds the `lava-broonie` lab along with other labs: @@ -84,13 +84,17 @@ scheduler-lava: <<: *scheduler container_name: 'kernelci-pipeline-scheduler-lava' command: - - './pipeline/scheduler.py' + - './src/scheduler.py' + - '--yaml-config=/home/kernelci/config' - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' - 'loop' + - '--name=scheduler_lava' - '--runtimes' - 'lava-collabora' - 'lava-collabora-staging' - 'lava-broonie' + extra_hosts: + - "host.docker.internal:host-gateway" ``` ### Jobs and devices specific to the lab diff --git a/docker-compose-kcidb.yaml b/docker-compose-kcidb.yaml deleted file mode 100644 index f04613926..000000000 --- a/docker-compose-kcidb.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -version: '3' - -services: - - kcidb: - container_name: 'kernelci-pipeline-kcidb' - image: 'ghcr.io/kernelci/staging-kernelci:pipeline' - env_file: ['.env'] - stop_signal: 'SIGINT' - networks: ['kcidb'] - command: - - '/usr/bin/env' - - 'python3' - - './src/send_kcidb.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' - - 'run' - - '--name=pipeline_kcidb' - volumes: - - './config:/home/kernelci/config' - - './data/kcidb:/home/kernelci/data/kcidb' - - './logs:/home/kernelci/logs' - restart: on-failure - postgres: - container_name: 'kernelci-pipeline-postgres-proxy' - image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy:latest' - command: 'kernelci-production:us-central1:postgresql2 -c/home/kernelci/data/kcidb/kernelci-production-ci-kernelci.json -a0.0.0.0' - networks: ['kcidb'] - volumes: - - './data/kcidb:/home/kernelci/data/kcidb' - restart: on-failure - -networks: - kcidb: - driver: bridge diff --git a/docker-compose-lava.yaml b/docker-compose-lava.yaml deleted file mode 100644 index cc67d40a4..000000000 --- a/docker-compose-lava.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -version: '3' - -services: - - lava-callback: - container_name: 'kernelci-pipeline-lava-callback' - networks: ['lava-callback'] - build: - context: 'docker/lava-callback' - env_file: ['.env'] - ports: - - '${LAVA_CALLBACK_PORT:-8100}:8000' - # With Werkzeug development server, not suitable for a public instance - command: - - '/usr/bin/env' - - 'python3' - - '/home/kernelci/pipeline/lava_callback.py' - volumes: - - './src:/home/kernelci/pipeline' - - './config:/home/kernelci/config' - - './data/ssh:/home/kernelci/data/ssh' - restart: on-failure - -networks: - lava-callback: - driver: bridge diff --git a/docker-compose-production.yaml b/docker-compose-production.yaml index 5cd0bc350..27a41a87e 100644 --- a/docker-compose-production.yaml +++ b/docker-compose-production.yaml @@ -3,8 +3,6 @@ # Copyright (C) 2023 Collabora Limited # Author: Guillaume Tucker -version: '3' - services: lava-callback: diff --git a/docker-compose.yaml b/docker-compose.yaml index ba17d85c6..e295f5cb3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,7 +4,11 @@ # Author: Guillaume Tucker # Author: Jeny Sadadia -version: '3' +x-config-arg: &config-arg + --yaml-config=/home/kernelci/config + +x-setting-arg: &setting-arg + --settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml} services: @@ -15,10 +19,10 @@ services: stop_signal: 'SIGINT' command: - './src/monitor.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'run' - volumes: &base-volumes -# - './src:/home/kernelci/pipeline' + volumes: - './config:/home/kernelci/config' - './logs:/home/kernelci/logs' extra_hosts: @@ -26,44 +30,36 @@ services: restart: on-failure result_summary: + <<: *base-service container_name: 'kernelci-pipeline-result-summary' - image: 'kernelci/staging-kernelci:pipeline' - env_file: ['.env'] - stop_signal: 'SIGINT' - entrypoint: + command: - './src/result_summary.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'run' - '--config=${CONFIG:-/home/kernelci/config/result-summary.yaml}' volumes: -# - './src:/home/kernelci/pipeline' - './config:/home/kernelci/config' - './data/output:/home/kernelci/data/output' - './logs:/home/kernelci/logs' - #restart: on-failure scheduler: &scheduler + <<: *base-service container_name: 'kernelci-pipeline-scheduler' - image: 'kernelci/staging-kernelci:pipeline' - env_file: ['.env'] - stop_signal: 'SIGINT' command: - './src/scheduler.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'loop' - '--runtimes=shell' - '--name=scheduler' volumes: -# - './src:/home/kernelci/pipeline' - './config:/home/kernelci/config' - './data/output:/home/kernelci/data/output' - './data/k8s-credentials/.kube:/home/kernelci/.kube' - './data/k8s-credentials/.config/gcloud:/home/kernelci/.config/gcloud' - './data/k8s-credentials/.azure:/home/kernelci/.azure' - './logs:/home/kernelci/logs' - extra_hosts: - - "host.docker.internal:host-gateway" - restart: on-failure scheduler-docker: <<: *scheduler @@ -72,26 +68,25 @@ services: #working_dir: /home/kernelci command: - './src/scheduler.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'loop' - '--runtimes=docker' - '--name=scheduler_docker' volumes: -# - './src:/home/kernelci/pipeline' - './config:/home/kernelci/config' - './data/output:/home/kernelci/data/output' - './.docker-env:/home/kernelci/.docker-env' - '/var/run/docker.sock:/var/run/docker.sock' # Docker-in-Docker - './logs:/home/kernelci/logs' - extra_hosts: - - "host.docker.internal:host-gateway" scheduler-lava: <<: *scheduler container_name: 'kernelci-pipeline-scheduler-lava' command: - './src/scheduler.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'loop' - '--name=scheduler_lava' - '--runtimes' @@ -104,129 +99,146 @@ services: - 'lava-cip' - 'lava-pengutronix' - 'lava-kontron' - extra_hosts: - - "host.docker.internal:host-gateway" scheduler-k8s: <<: *scheduler container_name: 'kernelci-pipeline-scheduler-k8s' - image: 'kernelci/staging-kernelci:pipeline' command: - './src/scheduler.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'loop' - '--name=scheduler_k8s' - '--runtimes' - 'k8s-gke-eu-west4' - 'k8s-all' - extra_hosts: - - "host.docker.internal:host-gateway" tarball: <<: *base-service container_name: 'kernelci-pipeline-tarball' command: - './src/tarball.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'run' - '--name=tarball' volumes: -# - './src:/home/kernelci/pipeline' - './config:/home/kernelci/config' - './data/ssh:/home/kernelci/data/ssh' - './data/src:/home/kernelci/data/src' - './data/output:/home/kernelci/data/output' - './logs:/home/kernelci/logs' - extra_hosts: - - "host.docker.internal:host-gateway" trigger: <<: *base-service container_name: 'kernelci-pipeline-trigger' command: - './src/trigger.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'run' - '--trees=kernelci,netdev-testing' - '--name=trigger' - extra_hosts: - - "host.docker.internal:host-gateway" regression_tracker: <<: *base-service container_name: 'kernelci-pipeline-regression_tracker' command: - - '/usr/bin/env' - - 'python3' - - '/home/kernelci/pipeline/src/regression_tracker.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - './src/regression_tracker.py' + - *config-arg + - *setting-arg - 'run' - extra_hosts: - - "host.docker.internal:host-gateway" test_report: <<: *base-service container_name: 'kernelci-pipeline-test_report' command: - - '/usr/bin/env' - - 'python3' - - '/home/kernelci/pipeline/src/test_report.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - './src/test_report.py' + - *config-arg + - *setting-arg - 'loop' - extra_hosts: - - "host.docker.internal:host-gateway" timeout: <<: *base-service container_name: 'kernelci-pipeline-timeout' command: - - '/usr/bin/env' - - 'python3' - - '/home/kernelci/pipeline/src/timeout.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - './src/timeout.py' + - *config-arg + - *setting-arg - 'run' - '--mode=timeout' - extra_hosts: - - "host.docker.internal:host-gateway" timeout-closing: <<: *base-service container_name: 'kernelci-pipeline-closing' command: - - '/usr/bin/env' - - 'python3' - - '/home/kernelci/pipeline/src/timeout.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - './src/timeout.py' + - *config-arg + - *setting-arg - 'run' - '--mode=closing' - extra_hosts: - - "host.docker.internal:host-gateway" timeout-holdoff: <<: *base-service container_name: 'kernelci-pipeline-holdoff' command: - - '/usr/bin/env' - - 'python3' - - '/home/kernelci/pipeline/src/timeout.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - './src/timeout.py' + - *config-arg + - *setting-arg - 'run' - '--mode=holdoff' - extra_hosts: - - "host.docker.internal:host-gateway" patchset: <<: *base-service container_name: 'kernelci-pipeline-patchset' command: - './src/patchset.py' - - '--settings=${KCI_SETTINGS:-/home/kernelci/config/kernelci.toml}' + - *config-arg + - *setting-arg - 'run' - '--name=patchset' volumes: -# - './src:/home/kernelci/pipeline' - './config:/home/kernelci/config' - './data/ssh:/home/kernelci/data/ssh' - './data/src:/home/kernelci/data/src' - './data/output:/home/kernelci/data/output' - './logs:/home/kernelci/logs' + + lava-callback: + <<: *base-service + container_name: 'kernelci-pipeline-lava-callback' + image: 'kernelci/staging-kernelci:lava-callback' + ports: + - '${LAVA_CALLBACK_PORT:-8100}:8000' + # With Werkzeug development server, not suitable for a public instance + command: + - './src/lava_callback.py' + - *config-arg + - *setting-arg + volumes: + - './config:/home/kernelci/config' + - './data/ssh:/home/kernelci/data/ssh' + + kcidb: + <<: *base-service + container_name: 'kernelci-pipeline-kcidb' + command: + - './src/send_kcidb.py' + - *config-arg + - *setting-arg + - 'run' + - '--name=pipeline_kcidb' + volumes: + - './config:/home/kernelci/config' + - './data/kcidb:/home/kernelci/data/kcidb' + - './logs:/home/kernelci/logs' + + postgres: + container_name: 'kernelci-pipeline-postgres-proxy' + image: 'gcr.io/cloud-sql-connectors/cloud-sql-proxy:latest' + command: + - 'kernelci-production:us-central1:postgresql2' + - '-c/home/kernelci/data/kcidb/kernelci-production-ci-kernelci.json' + - '-a0.0.0.0' + volumes: + - './data/kcidb:/home/kernelci/data/kcidb' diff --git a/docker/kcidb/Dockerfile b/docker/kcidb/Dockerfile deleted file mode 100644 index 43b1562c4..000000000 --- a/docker/kcidb/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2021, 2022 Collabora Limited -# Author: Guillaume Tucker - -FROM kernelci/staging-kernelci -MAINTAINER "KernelCI TSC" - -# Install kcidb -RUN pip install git+https://github.com/kernelci/kcidb.git - -# Install logspec -RUN pip install git+https://github.com/kernelci/logspec.git@3421f63da61d80d00a4497a915d19520285f5020 diff --git a/docker/lava-callback/Dockerfile b/docker/lava-callback/Dockerfile deleted file mode 100644 index f4b0f7cf2..000000000 --- a/docker/lava-callback/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM kernelci/staging-kernelci - -WORKDIR /home/kernelci/callback -COPY requirements.txt . - -USER root -RUN pip3 install -r /home/kernelci/callback/requirements.txt - -USER kernelci -WORKDIR /home/kernelci/ diff --git a/docker/lava-callback/requirements.txt b/docker/lava-callback/requirements.txt deleted file mode 100644 index 0ede9cef3..000000000 --- a/docker/lava-callback/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -uwsgi==2.0.22 -uvicorn==0.30.1 -fastapi==0.111.0 -pyjwt==2.8.0 diff --git a/docker/pipeline/Dockerfile.production b/docker/pipeline/Dockerfile.production deleted file mode 100644 index 957efb40e..000000000 --- a/docker/pipeline/Dockerfile.production +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -FROM kernelci/k8s:kernelci -MAINTAINER "KernelCI TSC" - -ARG git_url=https://github.com/kernelci/kernelci-pipeline.git -ARG git_rev=main -RUN git clone --depth=1 $git_url pipeline -WORKDIR pipeline -RUN git fetch origin $git_rev -RUN git checkout FETCH_HEAD diff --git a/src/lava_callback.py b/src/lava_callback.py index 792925022..fe25345f9 100755 --- a/src/lava_callback.py +++ b/src/lava_callback.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # SPDX-License-Identifier: LGPL-2.1-or-later # # Copyright (C) 2023-2025 Collabora Limited