diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 776ac2800..76df515d3 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -1965,6 +1965,9 @@ spec: description: Disable web container's nginx ipv6 listener type: boolean default: false + idle_deployment: + description: Scale down deployments to put AWX into an idle state + type: boolean metrics_utility_enabled: description: Enable metrics utility type: boolean diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index c7f8cc7d4..f5ee7a7bb 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -1155,6 +1155,13 @@ spec: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true + - description: Scale down deployments to put AWX into an idle state + displayName: Idle AWX + path: idle_deployment + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - urn:alm:descriptor:com.tectonic.ui:hidden statusDescriptors: - description: Route to access the instance deployed displayName: URL diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index d6dbed00c..e9933efa3 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -505,3 +505,6 @@ nginx_worker_cpu_affinity: 'auto' nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}" extra_settings_files: {} + +# idle_deployment - Scale down deployments to put AWX into an idle state +idle_deployment: false diff --git a/roles/installer/tasks/database.yml b/roles/installer/tasks/database.yml new file mode 100644 index 000000000..f3a60af21 --- /dev/null +++ b/roles/installer/tasks/database.yml @@ -0,0 +1,167 @@ +--- +- name: Get database configuration + include_tasks: database_configuration.yml + +# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades. +# So we have to take that into account and preferentially set the most recent one. +- name: Get the old postgres pod (N-1) + k8s_info: + kind: Pod + namespace: "{{ ansible_operator_meta.namespace }}" + field_selectors: + - status.phase=Running + register: _running_pods + +- block: + - name: Filter pods by name + set_fact: + filtered_old_postgres_pods: "{{ _running_pods.resources | + selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') | + rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') | + list }}" + + # Sort pods by name in reverse order (most recent PG version first) and set + - name: Set info for previous postgres pod + set_fact: + sorted_old_postgres_pods: "{{ filtered_old_postgres_pods | + sort(attribute='metadata.name') | + reverse | list }}" + when: filtered_old_postgres_pods | length + + + - name: Set info for previous postgres pod + set_fact: + old_postgres_pod: "{{ sorted_old_postgres_pods | first }}" + when: filtered_old_postgres_pods | length + when: _running_pods.resources | length + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "{{ kind }}" + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ ansible_operator_meta.namespace }}" + register: this_awx + +# If this deployment has been upgraded before or if upgrade has already been started, set this var +- name: Set previous PG version var + set_fact: + _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}" + when: + - this_awx['resources'][0] is defined + - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']" + +- name: Check if postgres pod is running an older version + block: + - name: Get old PostgreSQL version + k8s_exec: + namespace: "{{ ansible_operator_meta.namespace }}" + pod: "{{ old_postgres_pod['metadata']['name'] }}" + command: | + bash -c """ + if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then + cat "{{ _postgres_data_path }}/PG_VERSION" + elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then + cat '/var/lib/postgresql/data/pgdata/PG_VERSION' + fi + """ + register: _old_pg_version + + - debug: + msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---" + + - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable + include_tasks: upgrade_postgres.yml + when: + - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version + when: + - managed_database + - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version | int < supported_pg_version, true) + - old_postgres_pod | length # If empty, then old pg pod has been removed and we can assume the upgrade is complete + +- block: + - name: Create Database if no database is specified + k8s: + apply: true + definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}" + register: create_statefulset_result + + - name: Scale down Deployment for migration + include_tasks: scale_down_deployment.yml + when: create_statefulset_result.changed + + rescue: + - name: Scale down Deployment for migration + include_tasks: scale_down_deployment.yml + + - name: Scale down PostgreSQL statefulset for migration + kubernetes.core.k8s_scale: + api_version: apps/v1 + kind: StatefulSet + name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}" + namespace: "{{ ansible_operator_meta.namespace }}" + replicas: 0 + wait: yes + + - name: Remove PostgreSQL statefulset for upgrade + k8s: + state: absent + api_version: apps/v1 + kind: StatefulSet + name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}" + namespace: "{{ ansible_operator_meta.namespace }}" + wait: yes + when: create_statefulset_result.error == 422 + + - name: Recreate PostgreSQL statefulset with updated values + k8s: + apply: true + definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}" + when: managed_database + +- name: Set Default label selector for custom resource generated postgres + set_fact: + postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}" + when: postgres_label_selector is not defined + +- name: Get the postgres pod information + k8s_info: + kind: Pod + namespace: "{{ ansible_operator_meta.namespace }}" + label_selectors: + - "{{ postgres_label_selector }}" + field_selectors: + - status.phase=Running + register: postgres_pod + +- name: Wait for Database to initialize if managed DB + k8s_info: + kind: Pod + namespace: '{{ ansible_operator_meta.namespace }}' + label_selectors: + - "{{ postgres_label_selector }}" + field_selectors: + - status.phase=Running + register: postgres_pod + until: + - "postgres_pod['resources'] | length" + - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" + - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true" + delay: 5 + retries: 60 + when: managed_database + +- name: Look up details for this deployment + k8s_info: + api_version: "{{ api_version }}" + kind: "{{ kind }}" + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ ansible_operator_meta.namespace }}" + register: this_awx + +- name: Migrate data from old Openshift instance + import_tasks: migrate_data.yml + when: + - old_pg_config['resources'] is defined + - old_pg_config['resources'] | length + - this_awx['resources'][0]['status']['migratedFromSecret'] is not defined diff --git a/roles/installer/tasks/database_configuration.yml b/roles/installer/tasks/database_configuration.yml index 31742d182..d5ee4239a 100644 --- a/roles/installer/tasks/database_configuration.yml +++ b/roles/installer/tasks/database_configuration.yml @@ -106,167 +106,3 @@ - name: Set database as managed set_fact: managed_database: "{{ pg_config['resources'][0]['data']['type'] | default('') | b64decode == 'managed' }}" - -# It is possible that N-2 postgres pods may still be present in the namespace from previous upgrades. -# So we have to take that into account and preferentially set the most recent one. -- name: Get the old postgres pod (N-1) - k8s_info: - kind: Pod - namespace: "{{ ansible_operator_meta.namespace }}" - field_selectors: - - status.phase=Running - register: _running_pods - -- block: - - name: Filter pods by name - set_fact: - filtered_old_postgres_pods: "{{ _running_pods.resources | - selectattr('metadata.name', 'match', ansible_operator_meta.name + '-postgres.*-0') | - rejectattr('metadata.name', 'search', '-' + supported_pg_version | string + '-0') | - list }}" - - # Sort pods by name in reverse order (most recent PG version first) and set - - name: Set info for previous postgres pod - set_fact: - sorted_old_postgres_pods: "{{ filtered_old_postgres_pods | - sort(attribute='metadata.name') | - reverse | list }}" - when: filtered_old_postgres_pods | length - - - - name: Set info for previous postgres pod - set_fact: - old_postgres_pod: "{{ sorted_old_postgres_pods | first }}" - when: filtered_old_postgres_pods | length - when: _running_pods.resources | length - -- name: Look up details for this deployment - k8s_info: - api_version: "{{ api_version }}" - kind: "{{ kind }}" - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ ansible_operator_meta.namespace }}" - register: this_awx - -# If this deployment has been upgraded before or if upgrade has already been started, set this var -- name: Set previous PG version var - set_fact: - _previous_upgraded_pg_version: "{{ this_awx['resources'][0]['status']['upgradedPostgresVersion'] | default(false) }}" - when: - - this_awx['resources'][0] is defined - - "'upgradedPostgresVersion' in this_awx['resources'][0]['status']" - -- name: Check if postgres pod is running an older version - block: - - name: Get old PostgreSQL version - k8s_exec: - namespace: "{{ ansible_operator_meta.namespace }}" - pod: "{{ old_postgres_pod['metadata']['name'] }}" - command: | - bash -c """ - if [ -f "{{ _postgres_data_path }}/PG_VERSION" ]; then - cat "{{ _postgres_data_path }}/PG_VERSION" - elif [ -f '/var/lib/postgresql/data/pgdata/PG_VERSION' ]; then - cat '/var/lib/postgresql/data/pgdata/PG_VERSION' - fi - """ - register: _old_pg_version - - - debug: - msg: "--- Upgrading from {{ old_postgres_pod['metadata']['name'] | default('NONE')}} Pod ---" - - - name: Upgrade data dir from old Postgres to {{ supported_pg_version }} if applicable - include_tasks: upgrade_postgres.yml - when: - - (_old_pg_version.stdout | default(0) | int ) < supported_pg_version - when: - - managed_database - - (_previous_upgraded_pg_version | default(false)) | ternary(_previous_upgraded_pg_version | int < supported_pg_version, true) - - old_postgres_pod | length # If empty, then old pg pod has been removed and we can assume the upgrade is complete - -- block: - - name: Create Database if no database is specified - k8s: - apply: true - definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}" - register: create_statefulset_result - - - name: Scale down Deployment for migration - include_tasks: scale_down_deployment.yml - when: create_statefulset_result.changed - - rescue: - - name: Scale down Deployment for migration - include_tasks: scale_down_deployment.yml - - - name: Scale down PostgreSQL statefulset for migration - kubernetes.core.k8s_scale: - api_version: apps/v1 - kind: StatefulSet - name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}" - namespace: "{{ ansible_operator_meta.namespace }}" - replicas: 0 - wait: yes - - - name: Remove PostgreSQL statefulset for upgrade - k8s: - state: absent - api_version: apps/v1 - kind: StatefulSet - name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}" - namespace: "{{ ansible_operator_meta.namespace }}" - wait: yes - when: create_statefulset_result.error == 422 - - - name: Recreate PostgreSQL statefulset with updated values - k8s: - apply: true - definition: "{{ lookup('template', 'statefulsets/postgres.yaml.j2') }}" - when: managed_database - -- name: Set Default label selector for custom resource generated postgres - set_fact: - postgres_label_selector: "app.kubernetes.io/instance=postgres-{{ supported_pg_version }}-{{ ansible_operator_meta.name }}" - when: postgres_label_selector is not defined - -- name: Get the postgres pod information - k8s_info: - kind: Pod - namespace: "{{ ansible_operator_meta.namespace }}" - label_selectors: - - "{{ postgres_label_selector }}" - field_selectors: - - status.phase=Running - register: postgres_pod - -- name: Wait for Database to initialize if managed DB - k8s_info: - kind: Pod - namespace: '{{ ansible_operator_meta.namespace }}' - label_selectors: - - "{{ postgres_label_selector }}" - field_selectors: - - status.phase=Running - register: postgres_pod - until: - - "postgres_pod['resources'] | length" - - "postgres_pod['resources'][0]['status']['phase'] == 'Running'" - - "postgres_pod['resources'][0]['status']['containerStatuses'][0]['ready'] == true" - delay: 5 - retries: 60 - when: managed_database - -- name: Look up details for this deployment - k8s_info: - api_version: "{{ api_version }}" - kind: "{{ kind }}" - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ ansible_operator_meta.namespace }}" - register: this_awx - -- name: Migrate data from old Openshift instance - import_tasks: migrate_data.yml - when: - - old_pg_config['resources'] is defined - - old_pg_config['resources'] | length - - this_awx['resources'][0]['status']['migratedFromSecret'] is not defined diff --git a/roles/installer/tasks/idle_deployment.yml b/roles/installer/tasks/idle_deployment.yml new file mode 100644 index 000000000..5cc5ff51c --- /dev/null +++ b/roles/installer/tasks/idle_deployment.yml @@ -0,0 +1,34 @@ +--- +- name: Scale down AWX Deployments + kubernetes.core.k8s: + state: present + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: "{{ item }}" + namespace: "{{ ansible_operator_meta.namespace }}" + spec: + replicas: 0 + loop: + - '{{ ansible_operator_meta.name }}-task' + - '{{ ansible_operator_meta.name }}-web' + +- name: Get database configuration + include_tasks: database_configuration.yml + +- name: Scale down PostgreSQL Statefulset + kubernetes.core.k8s: + state: present + definition: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: "{{ ansible_operator_meta.name }}-postgres-{{ supported_pg_version }}" + namespace: "{{ ansible_operator_meta.namespace }}" + spec: + replicas: 0 + when: managed_database + +- name: End Playbook + ansible.builtin.meta: end_play diff --git a/roles/installer/tasks/install.yml b/roles/installer/tasks/install.yml index b0099eb9d..bc1d7e919 100644 --- a/roles/installer/tasks/install.yml +++ b/roles/installer/tasks/install.yml @@ -74,8 +74,8 @@ - name: Include set_images tasks include_tasks: set_images.yml -- name: Include database configuration tasks - include_tasks: database_configuration.yml +- name: Include Database tasks + include_tasks: database.yml - name: Load Route TLS certificate include_tasks: load_route_tls_secret.yml diff --git a/roles/installer/tasks/main.yml b/roles/installer/tasks/main.yml index dffe35f75..3440effc0 100644 --- a/roles/installer/tasks/main.yml +++ b/roles/installer/tasks/main.yml @@ -1,4 +1,8 @@ --- +- name: Idle AWX + include_tasks: idle_deployment.yml + when: idle_deployment | bool + - name: Check for presence of old awx Deployment k8s_info: api_version: apps/v1