From 2a74bb5377a1a3d15941fba4367b0493ec7a5a40 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 21 Jan 2025 15:46:38 +0000 Subject: [PATCH 1/9] feat: add required collections and roles for GitLab CI This commit adds the required collections and roles for GitLab CI to the `etc/kayobe/ansible/requirements.yml` file. Note: this will need changing before merging but is required until the roles and collections been merged and released. --- etc/kayobe/ansible/requirements.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 569ec172f..b8b469c91 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -8,10 +8,14 @@ collections: version: 0.0.13 - name: stackhpc.pulp version: 0.5.5 - - name: stackhpc.hashicorp - version: 2.5.1 - - name: stackhpc.kayobe_workflows - version: 1.1.0 + # (jackhodgkiss) Update once role is merged into collection. + - name: https://github.com/stackhpc/ansible-collection-hashicorp.git + type: git + version: e94bed3838ff5116d7f60d57bcd833153d2f4e9c + # (jackhodgkiss) Update once role is merged into collection. + - name: https://github.com/stackhpc/ansible-collection-kayobe-workflows.git + type: git + version: ca2d6d93123d78c5f829385773a7cfb5adcebadf roles: - src: stackhpc.vxlan - name: ansible-lockdown.ubuntu22_cis @@ -29,3 +33,7 @@ roles: version: 1.18.5 - src: https://github.com/stackhpc/ansible-role-docker.git name: geerlingguy.docker + # (jackhodgkiss) Update once patch is merged and released upstream. + - src: https://github.com/stackhpc/ansible-gitlab-runner + name: riemers.gitlab-runner + version: use-ansible-facts From 156c714d31fdf0eab5baee30ab2e5349cba322f1 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 21 Jan 2025 15:49:18 +0000 Subject: [PATCH 2/9] feat: add group definitions for gitlab runners and writers --- .../inventory/group_vars/gitlab-runners/runners.yml | 9 +++++++++ .../inventory/group_vars/gitlab-writers/writer.yml | 8 ++++++++ etc/kayobe/inventory/groups | 7 +++++++ 3 files changed, 24 insertions(+) create mode 100644 etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml create mode 100644 etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml diff --git a/etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml b/etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml new file mode 100644 index 000000000..346acef27 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml @@ -0,0 +1,9 @@ +--- +# Configuration of GitLab runners using riemers.gitlab-runner should go here. +# See documentation for more information +# https://github.com/riemers/ansible-gitlab-runner +# https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-2024.1/configuration/ci-cd.html + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml b/etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml new file mode 100644 index 000000000..ac2c962a7 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml @@ -0,0 +1,8 @@ +--- +# Configuration of GitLab pipelines generated with stackhpc.kayobe_workflows.gitlab should go here. +# See documentation for more information +# https://github.com/stackhpc/ansible-collection-kayobe-workflows/blob/main/roles/gitlab/README.md + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index 1028c2ca9..a2f2b0215 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -38,6 +38,12 @@ overcloud [github-writer] localhost +[gitlab-runners] +# Empty group to provide declaration of gitlab-runner group. + +[gitlab-writer] +localhost + ############################################################################### # Overcloud groups. @@ -87,6 +93,7 @@ network monitoring storage compute +infra-vms [docker-registry:children] # Hosts in this group will have a Docker Registry deployed. This group should From 609b6bedd9d04e316fda14d7b1800e2fca09905d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 21 Jan 2025 15:49:54 +0000 Subject: [PATCH 3/9] feat: add playbooks for deploying runners and OpenBao automation --- etc/kayobe/ansible/deploy-gitlab-runner.yml | 25 +++++++ .../deploy-openbao-kayobe-automation.yml | 70 +++++++++++++++++++ etc/kayobe/ansible/write-gitlab-pipelines.yml | 7 ++ 3 files changed, 102 insertions(+) create mode 100644 etc/kayobe/ansible/deploy-gitlab-runner.yml create mode 100644 etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml create mode 100644 etc/kayobe/ansible/write-gitlab-pipelines.yml diff --git a/etc/kayobe/ansible/deploy-gitlab-runner.yml b/etc/kayobe/ansible/deploy-gitlab-runner.yml new file mode 100644 index 000000000..7525b6949 --- /dev/null +++ b/etc/kayobe/ansible/deploy-gitlab-runner.yml @@ -0,0 +1,25 @@ +--- +- name: Deploy GitLab runners + hosts: gitlab-runners + become: true + pre_tasks: + - name: Ensure /opt/.docker folder exists + ansible.builtin.file: + path: /opt/.docker + state: directory + + - name: Ensure docker/config.json exists for runner + ansible.builtin.file: + content: | + { + "auths": { + "{{ pulp_url | regex_replace('^https?://|^http?://', '') }}": { + "auth": "{{ (pulp_username + ':' + pulp_password) | b64encode }}" + } + } + } + dest: /opt/.docker/config.json + mode: "0600" + + roles: + - name: riemers.gitlab-runner diff --git a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml new file mode 100644 index 000000000..93d8eee15 --- /dev/null +++ b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml @@ -0,0 +1,70 @@ +--- +- name: Deploy OpenBao on the runners + any_errors_fatal: true + gather_facts: true + hosts: github-runners,gitlab-runners + tasks: + - name: Set a fact about the virtualenv on the remote system + set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure /opt/kayobe/vault exists + file: + path: /opt/kayobe/vault + state: directory + become: true + + - import_role: + name: stackhpc.hashicorp.openbao + vars: + openbao_config_dir: "/opt/kayobe/vault" + openbao_cluster_name: "kayobe-automation" + copy_self_signed_ca: false + openbao_write_keys_file: true + openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/kayobe-automation-keys.json" + + - name: Include OpenBao keys + include_vars: + file: "{{ kayobe_env_config_path }}/vault/kayobe-automation-keys.json" + name: openbao_keys + tags: always + + - import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: "{{ openbao_api_addr }}" + vault_unseal_token: "{{ openbao_keys.root_token }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + vault_unseal_verify: false + environment: + https_proxy: '' + + - name: Create secret store + hashivault_secret_engine: + name: kayobe-automation + backend: kv + url: "{{ openbao_api_addr }}" + token: "{{ openbao_keys.root_token }}" + + - name: Ensure secret store is present + community.hashi_vault.vault_write: + url: "{{ openbao_api_addr }}" + token: "{{ openbao_keys.root_token }}" + path: kayobe-automation/{{ kayobe_environment }} + data: + kayobe_vault_password: "{{ kolla_ansible_vault_password }}" + kayobe_automation_ssh_private_key: "{{ lookup('ansible.builtin.file', '{{ ssh_private_key_path }}') }}" + kayobe_public_openrc: "{{ lookup('ansible.builtin.file', '{{ kolla_config_path }}/public-openrc.sh') }}" + tags: add_secrets diff --git a/etc/kayobe/ansible/write-gitlab-pipelines.yml b/etc/kayobe/ansible/write-gitlab-pipelines.yml new file mode 100644 index 000000000..c85e6adcd --- /dev/null +++ b/etc/kayobe/ansible/write-gitlab-pipelines.yml @@ -0,0 +1,7 @@ +--- +- name: Write Kayobe Automation Pipeline for GitLab + hosts: gitlab-writer + vars: + gitlab_output_directory: "{{ kayobe_config_path }}/../../" + roles: + - stackhpc.kayobe_workflows.gitlab From d0de34bfb4000a44a843f8ddff999d0ab330b935 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 21 Jan 2025 15:50:15 +0000 Subject: [PATCH 4/9] feat: update `ci-cd` documentation Includes steps for deploying GitLab runners, generating pipelines and deploying OpenBao for secret management. --- doc/source/configuration/ci-cd.rst | 202 ++++++++++++++++++++++++++++- 1 file changed, 201 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 6e495c2e8..a07b79bd5 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -5,12 +5,14 @@ CI/CD Concepts ======== -The CI/CD system developed for managing Kayobe based OpenStack clouds is composed of three main components; workflows, runners and kayobe automation. +The CI/CD system developed for managing Kayobe based OpenStack clouds is composed of four main components; workflows, runners, OpenBao and kayobe automation. Firstly, the workflows are files which describe a series of tasks to be performed in relation to the deployed cloud. These workflows are executed on request, on schedule or in response to an event such as a pull request being opened. The workflows are designed to carry out various day-to-day activites such as; running Tempest tests, configuring running services or displaying the change to configuration files if a pull request is merged. Secondly, in order for the workflows to run against a cloud we would need private runners present within the cloud positioned in such a way they can reach the internal network and public API. Deployment of private runners is supported by all major providers with the use of community developed Ansible roles. +Thirdly, OpenBao is used to store secrets on the same virtual machine the runners are hosted within. +This provides a secure way of storing secrets and variables which can be accessed by the runners when executing workflows and ensures that secrets never have to leave the cloud. Finally, due to the requirement that we support various different platforms tooling in the form of `Kayobe automation `__ was developed. This tooling is not tied to any single CI/CD platform as all tasks are a series of shell script and Ansible playbooks which are designed to run in a purpose build kayobe container. This is complemented by the use of an Ansible collection known as `stackhpc.kayobe_workflows `__ which aims to provide users with a quick and easy way of customising all workflows to fit within a customer's cloud. @@ -42,6 +44,12 @@ These services will listen for jobs which have been tagged appropriately and dis The runners will need to be deployed using existing roles and playbooks whereby the binary/package is downloaded and registered using a special token. In some deployments runner hosts can be shared between environments however this is not always true and dedicated hosts will need to be used for each environment you intend to deploy kayobe automation within. +OpenBao +------- + +OpenBao is recommended when deploying kayobe automation to achieve a simple and secure way of storing secrets. +OpenBao can easily be configured to hold the secrets for all environments and only permit access to the runners which require them utilising different authorisation mechanisms such as GitLab's JWT (JSON Web Token). + GitHub Actions ================= @@ -181,3 +189,195 @@ Sometimes the kayobe docker image must be rebuilt the reasons for this include b * Update kolla-ansible * UID/GID collision when deploying workflows to a new environment * Prior to deployment of new a OpenStack release + +GitLab Pipelines +================ + +To enable CI/CD where GitLab Pipelines is used please follow the steps described below starting with the deployment of the runners. + +Runner Deployment +----------------- + +1. Identify a suitable host for hosting the runners. + Ideally an infra-vm would be deployed to allow for easily compartmentalising the runners from the rest of the environment. + 8 VCPUs and 16GB of RAM is recommended for the guest machine however this may need to be adjusted for larger deployments. + Whether the host is in an infra-vm or not it will need access to the :code:`admin_network` or :code:`provision_oc_network`, :code:`public_network` and the :code:`pulp registry` on the seed. + The steps will assume that an infra-vm will be used for the purpose of hosting the runners. + +2. Edit the environment's :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/hosts` to define the host(s) that will host the runners. + +.. code-block:: ini + + [github-runners] + gitlab-runner-01 + +4. Provide all the relevant Kayobe :code:`group_vars` for :code:`gitlab-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/gitlab-runners` + * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined + * `network-interfaces` + +5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/gitlab-runners/runners.yml`` file which will contain the variables required to deploy a series of runners. + Below is an example of how GitLab runners can be configured for deployment. + In this example we have two runners, one for production and one for staging and will both be deployed on the same host. + This might not be possible for all deployments as multiple environments may require different runners as no single runner can serve all environments. + Note a GitLab runner can run multiple jobs concurrently so deploying a single runner per environment is recommended. + +.. code-block:: yaml + + --- + gitlab_runner_coordinator_url: "https://gitlab.example.com" + gitlab_runner_runners: + - name: "Kayobe Automation Runner [Production] #1" + executor: docker + docker_image: 'alpine' + token: "{{ secrets_gitlab_production_runner_token }}" + env_vars: + - "GIT_CONFIG_COUNT=1" + - "GIT_CONFIG_KEY_0=safe.directory" + - "GIT_CONFIG_VALUE_0=*" + tags: + - kayobe + - openstack + - production + docker_volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "/opt/.docker/config.json:/root/.docker/config.json:ro" + - "/cache" + extra_configs: + runners.docker: + network_mode: host + - name: "Kayobe Automation Runner [Staging] #1" + executor: docker + docker_image: 'alpine' + token: "{{ secrets_gitlab_staging_runner_token }}" + env_vars: + - "GIT_CONFIG_COUNT=1" + - "GIT_CONFIG_KEY_0=safe.directory" + - "GIT_CONFIG_VALUE_0=*" + tags: + - kayobe + - openstack + - staging + docker_volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "/opt/.docker/config.json:/root/.docker/config.json:ro" + - "/cache" + extra_configs: + runners.docker: + network_mode: host + +6. Obtain a runner token for each runner that is required for deployment. + This token can be obtained by visiting the GitLab project -> Settings -> CI/CD -> Runners -> New project runner -> Complete the form and copy the token. + Once the token has been obtained, add it to :code:`secrets.yml` under :code:`secrets_gitlab_production_runner_token` and :code:`secrets_gitlab_staging_runner_token` + +7. Deploy the infra-vm + +.. code-block:: bash + + kayobe infra vm provision --limit gitlab-runner-01 + +8. Perform a host configure against the infra-vm + +.. code-block:: bash + + kayobe infra vm host configure --limit gitlab-runner-01 + +9. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-gitlab-runner.yml` + +10. Check runners have registered properly by visiting the repository's :code:`CI/CD` tab -> :code:`Runners` + +11. The contents of :code:`/opt/.docker/config.json` on the runner should be added to GitLab CI/CD settings as a sercret variable. + This is required to allow the runners to pull images from the registry. + Visit the GitLab project -> Settings -> CI/CD -> Variables -> Add a new variable with the key :code:`DOCKER_CONFIG_JSON` and the value of the contents of :code:`/opt/.docker/config.json` + +OpenBao Deployment +------------------ + +OpenBao must be installed on the same host as the runners. +If you have multiple environments that each have the own runners then OpenBao must be installed on each host. +However, if you have a single host that is shared between environments then OpenBao only needs to be installed once and can be achieved by running the following playbook. + +.. code-block:: bash + + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-openbao-kayobe-automation.yml + +.. note:: + + If you are sharing OpenBao between environments then you will need to rerun the playbook under each environment to ensure that the correct secrets are available to the runners. + You may use :code:`--tags add_secrets` to skip the deployment within other environments. + For this to work you will need to copy :code:`vault/vault-automation-keys.json` from the first environment to the other environments in addition to copying the host definition of the gitlab runner add network IP. + +Once the above playbook has been applied you need to grab the root token from :code:`vault/vault-automation-keys.json` as you will need this to enable JWT support. +This would also be an opportune time to encrypt the :code:`vault/vault-automation-keys.json` to protect the contents. + +In order to enable JWT support the following steps must be carried out within the openbao container on the runner host. + +1. SSH into the runner host + +2. Run :code:`sudo docker exec -it bao sh` + +3. Run :code:`export BAO_AUTH_ADDR=http://127.0.0.1:8200` + +4. Run :code:`bao login` and use root token + +5. Run the following to enable and configure JWT support + +.. note:: + + The following steps are an example and should be adapted to suit your deployment. + For example project_id within the gitlab role will need ID of the project that the runners are registered against. + This can acquired by visiting the project -> Settings -> General -> General project settings -> Project ID. + +.. code-block:: bash + + bao auth enable jwt + bao policy write kayobe-automation - <`__. +Following the instructions in the documentation will allow you to customise the workflows to fit within your deployment. +For example disabling jobs that might not be relevant such as physical network configuration or overcloud host provision in clouds where this is absent. +Also consider the impact runbooks might have as the runbooks are designed with a particular cloud in mind and may not be suitable for all deployments such as hyperconverged deployments with Ceph on hypervisors. + +2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-gitlab-pipelines.yml` + +3. Commit and push all newly generated pipelines found under root of the repository. + +Things to consider +================== + +- Adjust General Pipeline settings by visiting the project -> Settings -> CI/CD -> General pipelines + - Disable :code:`Public Pipelines` + - Disable :code:`Auto-cancel redundant pipelines` + - Disable :code:`Prevent outdated deployment jobs` + - Increase :code:`Timeout` to :code:`12h` + +- Disable Auto DevOps in the GitLab project settings by visiting the project -> Settings -> CI/CD -> Auto DevOps -> Disable Auto DevOps + +Sometimes the kayobe docker image must be rebuilt the reasons for this include but are not limited to the following; + + * Change :code:`$KAYOBE_CONFIG_PATH/ansible/requirements.yml` + * Change to requirements.txt + * Update Kayobe + * Update kolla-ansible + * Prior to deployment of new a OpenStack release From beba0779ba0520231a63f9314eac32d328befbf0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 22 Jan 2025 09:40:32 +0000 Subject: [PATCH 5/9] fix: linting issues --- doc/source/configuration/ci-cd.rst | 4 ++-- .../ansible/deploy-openbao-kayobe-automation.yml | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index a07b79bd5..b83acfaf0 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -220,7 +220,7 @@ Runner Deployment In this example we have two runners, one for production and one for staging and will both be deployed on the same host. This might not be possible for all deployments as multiple environments may require different runners as no single runner can serve all environments. Note a GitLab runner can run multiple jobs concurrently so deploying a single runner per environment is recommended. - + .. code-block:: yaml --- @@ -357,7 +357,7 @@ GitLab Pipelines 1. Edit :code:`${KAYOBE_CONFIG_PATH}/inventory/group_vars/gitlab-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.gitlab `__. Following the instructions in the documentation will allow you to customise the workflows to fit within your deployment. For example disabling jobs that might not be relevant such as physical network configuration or overcloud host provision in clouds where this is absent. -Also consider the impact runbooks might have as the runbooks are designed with a particular cloud in mind and may not be suitable for all deployments such as hyperconverged deployments with Ceph on hypervisors. +Also consider the impact runbooks might have as the runbooks are designed with a particular cloud in mind and may not be suitable for all deployments such as hyperconverged deployments with Ceph on hypervisors. 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-gitlab-pipelines.yml` diff --git a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml index 93d8eee15..cd6e14b0a 100644 --- a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml +++ b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml @@ -5,7 +5,7 @@ hosts: github-runners,gitlab-runners tasks: - name: Set a fact about the virtualenv on the remote system - set_fact: + ansible.builtin.set_fact: virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" when: - ansible_python_interpreter is defined @@ -13,7 +13,7 @@ - not ansible_python_interpreter.startswith('/usr/bin/') - name: Ensure Python hvac module is installed - pip: + ansible.builtin.pip: name: hvac state: latest extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" @@ -21,12 +21,13 @@ become: "{{ virtualenv is not defined }}" - name: Ensure /opt/kayobe/vault exists - file: + ansible.builtin.file: path: /opt/kayobe/vault state: directory become: true - - import_role: + - name: Import OpenBao role + ansible.builtin.import_role: name: stackhpc.hashicorp.openbao vars: openbao_config_dir: "/opt/kayobe/vault" @@ -36,12 +37,13 @@ openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/kayobe-automation-keys.json" - name: Include OpenBao keys - include_vars: + ansible.builtin.include_vars: file: "{{ kayobe_env_config_path }}/vault/kayobe-automation-keys.json" name: openbao_keys tags: always - - import_role: + - name: Import Vault unseal role + ansible.builtin.import_role: name: stackhpc.hashicorp.vault_unseal vars: vault_api_addr: "{{ openbao_api_addr }}" @@ -52,7 +54,7 @@ https_proxy: '' - name: Create secret store - hashivault_secret_engine: + ansible.legacy.hashivault_secret_engine: name: kayobe-automation backend: kv url: "{{ openbao_api_addr }}" From beaedd3ebcda8352c40e047850d081f80f5401da Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 22 Jan 2025 12:51:35 +0000 Subject: [PATCH 6/9] fix: resolve issues against PR Improve the docs based on feedback provided. [skip ci] --- doc/source/configuration/ci-cd.rst | 11 ++++++++--- etc/kayobe/ansible/deploy-gitlab-runner.yml | 5 ++--- etc/kayobe/inventory/groups | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index b83acfaf0..2d19e73d7 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -6,15 +6,20 @@ Concepts ======== The CI/CD system developed for managing Kayobe based OpenStack clouds is composed of four main components; workflows, runners, OpenBao and kayobe automation. + Firstly, the workflows are files which describe a series of tasks to be performed in relation to the deployed cloud. These workflows are executed on request, on schedule or in response to an event such as a pull request being opened. + The workflows are designed to carry out various day-to-day activites such as; running Tempest tests, configuring running services or displaying the change to configuration files if a pull request is merged. Secondly, in order for the workflows to run against a cloud we would need private runners present within the cloud positioned in such a way they can reach the internal network and public API. Deployment of private runners is supported by all major providers with the use of community developed Ansible roles. + Thirdly, OpenBao is used to store secrets on the same virtual machine the runners are hosted within. This provides a secure way of storing secrets and variables which can be accessed by the runners when executing workflows and ensures that secrets never have to leave the cloud. + Finally, due to the requirement that we support various different platforms tooling in the form of `Kayobe automation `__ was developed. This tooling is not tied to any single CI/CD platform as all tasks are a series of shell script and Ansible playbooks which are designed to run in a purpose build kayobe container. + This is complemented by the use of an Ansible collection known as `stackhpc.kayobe_workflows `__ which aims to provide users with a quick and easy way of customising all workflows to fit within a customer's cloud. Currently we support the creation and deployment of workflows for GitHub with Gitlab support being actively worked upon. @@ -315,7 +320,7 @@ In order to enable JWT support the following steps must be carried out within th 2. Run :code:`sudo docker exec -it bao sh` -3. Run :code:`export BAO_AUTH_ADDR=http://127.0.0.1:8200` +3. Run :code:`export BAO_ADDR=http://127.0.0.1:8200` 4. Run :code:`bao login` and use root token @@ -349,7 +354,7 @@ In order to enable JWT support the following steps must be carried out within th EOF bao write auth/jwt/config \ jwks_url="https://gitlab.example.com/oauth/discovery/keys" \ - bound_issuer="https://gitlab.example.com" \ + bound_issuer="https://gitlab.example.com" GitLab Pipelines ---------------- @@ -374,7 +379,7 @@ Things to consider - Disable Auto DevOps in the GitLab project settings by visiting the project -> Settings -> CI/CD -> Auto DevOps -> Disable Auto DevOps -Sometimes the kayobe docker image must be rebuilt the reasons for this include but are not limited to the following; +Sometimes the kayobe docker image must be rebuilt. The reasons for this include but are not limited to the following; * Change :code:`$KAYOBE_CONFIG_PATH/ansible/requirements.yml` * Change to requirements.txt diff --git a/etc/kayobe/ansible/deploy-gitlab-runner.yml b/etc/kayobe/ansible/deploy-gitlab-runner.yml index 7525b6949..44a1002b8 100644 --- a/etc/kayobe/ansible/deploy-gitlab-runner.yml +++ b/etc/kayobe/ansible/deploy-gitlab-runner.yml @@ -9,7 +9,7 @@ state: directory - name: Ensure docker/config.json exists for runner - ansible.builtin.file: + ansible.builtin.copy: content: | { "auths": { @@ -20,6 +20,5 @@ } dest: /opt/.docker/config.json mode: "0600" - roles: - - name: riemers.gitlab-runner + - role: riemers.gitlab-runner diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index a2f2b0215..e7356f1a8 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -93,7 +93,7 @@ network monitoring storage compute -infra-vms +gitlab-runners [docker-registry:children] # Hosts in this group will have a Docker Registry deployed. This group should From 81974721027f272d439c37ab30506d71b09cf318 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 12 Feb 2025 15:40:19 +0000 Subject: [PATCH 7/9] feat: apply suggested changes Co-authored-by: Tom Clark <161027899+tomclark0@users.noreply.github.com> Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> --- doc/source/configuration/ci-cd.rst | 10 +++++----- .../ansible/deploy-openbao-kayobe-automation.yml | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 2d19e73d7..fc0f6a5f8 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -213,7 +213,7 @@ Runner Deployment .. code-block:: ini - [github-runners] + [gitlab-runners] gitlab-runner-01 4. Provide all the relevant Kayobe :code:`group_vars` for :code:`gitlab-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/gitlab-runners` @@ -292,7 +292,7 @@ Runner Deployment 11. The contents of :code:`/opt/.docker/config.json` on the runner should be added to GitLab CI/CD settings as a sercret variable. This is required to allow the runners to pull images from the registry. - Visit the GitLab project -> Settings -> CI/CD -> Variables -> Add a new variable with the key :code:`DOCKER_CONFIG_JSON` and the value of the contents of :code:`/opt/.docker/config.json` + Visit the GitLab project -> Settings -> CI/CD -> Variables -> Add a new variable with the key :code:`DOCKER_AUTH_CONFIG` and the value of the contents of :code:`/opt/.docker/config.json` OpenBao Deployment ------------------ @@ -309,10 +309,10 @@ However, if you have a single host that is shared between environments then Open If you are sharing OpenBao between environments then you will need to rerun the playbook under each environment to ensure that the correct secrets are available to the runners. You may use :code:`--tags add_secrets` to skip the deployment within other environments. - For this to work you will need to copy :code:`vault/vault-automation-keys.json` from the first environment to the other environments in addition to copying the host definition of the gitlab runner add network IP. + For this to work you will need to copy :code:`vault/kayobe-automation-keys.json` from the first environment to the other environments in addition to copying the host definition of the gitlab runner add network IP. -Once the above playbook has been applied you need to grab the root token from :code:`vault/vault-automation-keys.json` as you will need this to enable JWT support. -This would also be an opportune time to encrypt the :code:`vault/vault-automation-keys.json` to protect the contents. +Once the above playbook has been applied you need to grab the root token from :code:`vault/kayobe-automation-keys.json` as you will need this to enable JWT support. +This would also be an opportune time to encrypt the :code:`vault/kayobe-automation-keys.json` to protect the contents. In order to enable JWT support the following steps must be carried out within the openbao container on the runner host. diff --git a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml index cd6e14b0a..d45ae4ce7 100644 --- a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml +++ b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml @@ -26,6 +26,11 @@ state: directory become: true + - name: Ensure vault directory exists in environment + ansible.builtin.file: + path: {{ kayobe_env_config_path }}/vault + state: directory + become: true - name: Import OpenBao role ansible.builtin.import_role: name: stackhpc.hashicorp.openbao From f3b4fde9b565e4b3d488b8a035028eaef6dd48d2 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 12 Feb 2025 16:09:01 +0000 Subject: [PATCH 8/9] docs: update CI/CD documentation --- doc/source/configuration/ci-cd.rst | 10 ++++++++-- .../ansible/deploy-openbao-kayobe-automation.yml | 4 +++- .../{gitlab-writers => gitlab-writer}/writer.yml | 6 ++++++ 3 files changed, 17 insertions(+), 3 deletions(-) rename etc/kayobe/inventory/group_vars/{gitlab-writers => gitlab-writer}/writer.yml (68%) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index fc0f6a5f8..9f18059a7 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -219,6 +219,7 @@ Runner Deployment 4. Provide all the relevant Kayobe :code:`group_vars` for :code:`gitlab-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/gitlab-runners` * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined * `network-interfaces` + * `allocated IPs` 5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/gitlab-runners/runners.yml`` file which will contain the variables required to deploy a series of runners. Below is an example of how GitLab runners can be configured for deployment. @@ -271,7 +272,7 @@ Runner Deployment network_mode: host 6. Obtain a runner token for each runner that is required for deployment. - This token can be obtained by visiting the GitLab project -> Settings -> CI/CD -> Runners -> New project runner -> Complete the form and copy the token. + This token can be obtained by visiting the GitLab project -> Settings -> CI/CD -> Runners -> New project runner -> Complete the form including any tags used by the runners such as kayobe, openstack and environment_name. Once the token has been obtained, add it to :code:`secrets.yml` under :code:`secrets_gitlab_production_runner_token` and :code:`secrets_gitlab_staging_runner_token` 7. Deploy the infra-vm @@ -290,7 +291,7 @@ Runner Deployment 10. Check runners have registered properly by visiting the repository's :code:`CI/CD` tab -> :code:`Runners` -11. The contents of :code:`/opt/.docker/config.json` on the runner should be added to GitLab CI/CD settings as a sercret variable. +11. The contents of :code:`/opt/.docker/config.json` on the runner should be added to GitLab CI/CD settings as a sercret variable if GitLab version permits otherwise variable is fine. This is required to allow the runners to pull images from the registry. Visit the GitLab project -> Settings -> CI/CD -> Variables -> Add a new variable with the key :code:`DOCKER_AUTH_CONFIG` and the value of the contents of :code:`/opt/.docker/config.json` @@ -314,6 +315,10 @@ However, if you have a single host that is shared between environments then Open Once the above playbook has been applied you need to grab the root token from :code:`vault/kayobe-automation-keys.json` as you will need this to enable JWT support. This would also be an opportune time to encrypt the :code:`vault/kayobe-automation-keys.json` to protect the contents. +.. code-block:: bash + + ansible-vault encrypt vault/kayobe-automation-keys.json --vault-password-file ~/.vault.password + In order to enable JWT support the following steps must be carried out within the openbao container on the runner host. 1. SSH into the runner host @@ -362,6 +367,7 @@ GitLab Pipelines 1. Edit :code:`${KAYOBE_CONFIG_PATH}/inventory/group_vars/gitlab-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.gitlab `__. Following the instructions in the documentation will allow you to customise the workflows to fit within your deployment. For example disabling jobs that might not be relevant such as physical network configuration or overcloud host provision in clouds where this is absent. +If using multiple environments ensure that :code:`gitlab_kayobe_environments` is updated to reflect all environments present in the deployment. Also consider the impact runbooks might have as the runbooks are designed with a particular cloud in mind and may not be suitable for all deployments such as hyperconverged deployments with Ceph on hypervisors. 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-gitlab-pipelines.yml` diff --git a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml index d45ae4ce7..6f7fb98bf 100644 --- a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml +++ b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml @@ -28,9 +28,10 @@ - name: Ensure vault directory exists in environment ansible.builtin.file: - path: {{ kayobe_env_config_path }}/vault + path: "{{ kayobe_env_config_path }}/vault" state: directory become: true + - name: Import OpenBao role ansible.builtin.import_role: name: stackhpc.hashicorp.openbao @@ -75,3 +76,4 @@ kayobe_automation_ssh_private_key: "{{ lookup('ansible.builtin.file', '{{ ssh_private_key_path }}') }}" kayobe_public_openrc: "{{ lookup('ansible.builtin.file', '{{ kolla_config_path }}/public-openrc.sh') }}" tags: add_secrets + diff --git a/etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml b/etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml similarity index 68% rename from etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml rename to etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml index ac2c962a7..be2ce1c7e 100644 --- a/etc/kayobe/inventory/group_vars/gitlab-writers/writer.yml +++ b/etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml @@ -3,6 +3,12 @@ # See documentation for more information # https://github.com/stackhpc/ansible-collection-kayobe-workflows/blob/main/roles/gitlab/README.md +gitlab_output_directory: $KAYOBE_CONFIG_PATH/../../.gitlab/ + +gitlab_registry: "{{ pulp_url | regex_replace('^https?://|^http?://', '') }}" + +gitlab_openstack_release: "{{ openstack_release }}" + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 4d76e8484247db6312039fd1ab87748476144392 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 18 Feb 2025 10:11:58 +0000 Subject: [PATCH 9/9] feat: pin collections to branch names --- etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml | 1 - etc/kayobe/ansible/requirements.yml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml index 6f7fb98bf..195f23add 100644 --- a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml +++ b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml @@ -76,4 +76,3 @@ kayobe_automation_ssh_private_key: "{{ lookup('ansible.builtin.file', '{{ ssh_private_key_path }}') }}" kayobe_public_openrc: "{{ lookup('ansible.builtin.file', '{{ kolla_config_path }}/public-openrc.sh') }}" tags: add_secrets - diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index b8b469c91..564e2f6c6 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -11,11 +11,11 @@ collections: # (jackhodgkiss) Update once role is merged into collection. - name: https://github.com/stackhpc/ansible-collection-hashicorp.git type: git - version: e94bed3838ff5116d7f60d57bcd833153d2f4e9c + version: add-openbao # (jackhodgkiss) Update once role is merged into collection. - name: https://github.com/stackhpc/ansible-collection-kayobe-workflows.git type: git - version: ca2d6d93123d78c5f829385773a7cfb5adcebadf + version: add-gitlab-role roles: - src: stackhpc.vxlan - name: ansible-lockdown.ubuntu22_cis