From d3d94e5c52e403f88686a47901d9e2f7cababc7c Mon Sep 17 00:00:00 2001 From: Roman Hros Date: Fri, 23 Aug 2024 15:34:08 +0200 Subject: [PATCH] Update KaaS 'v1->v2' upgrade guide Signed-off-by: Roman Hros --- doc/usage/migrate-to-kaas-v2.md | 56 ++++-- .../files/kubernetes-manifests.d/cspo.yaml | 187 ++++++++++-------- terraform/files/template/clusterctl.yaml.tmpl | 4 +- 3 files changed, 146 insertions(+), 101 deletions(-) diff --git a/doc/usage/migrate-to-kaas-v2.md b/doc/usage/migrate-to-kaas-v2.md index b300a283..14caf98f 100644 --- a/doc/usage/migrate-to-kaas-v2.md +++ b/doc/usage/migrate-to-kaas-v2.md @@ -42,7 +42,12 @@ first, when you are on release < R6). ```bash kubectl patch secret -n ${CLUSTER_NAME} ${CLUSTER_NAME}-cloud-config -p '{"stringData":{"cloudName":"'"${PREFIX}-${CLUSTER_NAME}"'"}}' ``` -5. Create Cluster Stack: +5. Upgrade CAPI/CAPO: + ```bash + export CLUSTER_TOPOLOGY=true + clusterctl upgrade apply --infrastructure capo-system/openstack:v0.10.4 --core capi-system/cluster-api:v1.8.1 -b capi-kubeadm-bootstrap-system/kubeadm:v1.8.1 -c capi-kubeadm-control-plane-system/kubeadm:v1.8.1 + ``` +6. Create Cluster Stack: ```bash kubectl -n ${CLUSTER_NAME} apply -f - < Note: If you are using flavors with a disk, remove `controller_root_disk` and `worker_root_disk` variables ```bash cat << "EOF" | clusterctl generate yaml --config ~/${CLUSTER_NAME}/clusterctl.yaml | kubectl -n ${CLUSTER_NAME} apply -f - apiVersion: cluster.x-k8s.io/v1beta1 @@ -121,27 +131,39 @@ first, when you are on release < R6). value: ${OPENSTACK_SRVGRP_WORKER} - name: ssh_key value: ${OPENSTACK_SSH_KEY_NAME} - class: openstack-scs-1-28-v1 + class: openstack-scs-1-28-v2 version: ${KUBERNETES_VERSION} controlPlane: replicas: ${CONTROL_PLANE_MACHINE_COUNT} workers: machineDeployments: - - class: openstack-scs-1-28-v1-md-0-no1 + - class: default-worker name: "${PREFIX}-${CLUSTER_NAME}-md-0-no1" replicas: ${WORKER_MACHINE_COUNT} failureDomain: ${OPENSTACK_FAILURE_DOMAIN} EOF ``` - > Note: If you are using flavors with a disk, comment `controller_root_disk` and `worker_root_disk` variables -7. Fix metrics-server Cluster Addon: +9. Add back CAPO validation for updating the OpenStackCluster (add UPDATE operation to ValidatingWebhookConfiguration): ```bash - $ kubectl -n ${CLUSTER_NAME} get clusteraddon - NAME CLUSTER READY AGE REASON MESSAGE - cluster-addon-testcluster testcluster false 20m FailedToApplyObjects failed to successfully apply everything - $ KUBECONFIG=~/${CLUSTER_NAME}/${CLUSTER_NAME}.yaml kubectl delete deployment -n kube-system metrics-server - deployment.apps "metrics-server" deleted - $ kubectl -n ${CLUSTER_NAME} get clusteraddon - NAME CLUSTER READY AGE REASON MESSAGE - cluster-addon-testcluster testcluster true 25m + kubectl patch ValidatingWebhookConfiguration/capo-validating-webhook-configuration --type='json' -p='[{"op": "replace", "path": "/webhooks/0/rules/0/operations", "value":["CREATE", "UPDATE"]}]' ``` +10. Fix Cluster Addons: + ```bash + $ kubectl -n ${CLUSTER_NAME} get clusteraddon + NAME CLUSTER READY AGE REASON MESSAGE + cluster-addon-testcluster testcluster false 20m FailedToApplyObjects failed to successfully apply everything + # cannot update due to label selector changes - we can only delete old ones + $ KUBECONFIG=~/${CLUSTER_NAME}/${CLUSTER_NAME}.yaml kubectl delete deployment -n kube-system metrics-server + deployment.apps "metrics-server" deleted + $ KUBECONFIG=~/${CLUSTER_NAME}/${CLUSTER_NAME}.yaml kubectl delete daemonset -n kube-system openstack-cloud-controller-manager + daemonset.apps "openstack-cloud-controller-manager" deleted + $ kubectl -n ${CLUSTER_NAME} get clusteraddon + NAME CLUSTER READY AGE REASON MESSAGE + cluster-addon-testcluster testcluster true 25m + # names of csi-cinder-controller/nodeplugin changed to openstack-cinder-csi-controller/nodeplugin + # now we have duplicates which should be deleted + $ KUBECONFIG=~/${CLUSTER_NAME}/${CLUSTER_NAME}.yaml kubectl delete deployment -n kube-system csi-cinder-controllerplugin + deployment.apps "csi-cinder-controllerplugin" deleted + $ KUBECONFIG=~/${CLUSTER_NAME}/${CLUSTER_NAME}.yaml kubectl delete daemonset -n kube-system csi-cinder-nodeplugin + daemonset.apps "csi-cinder-nodeplugin" deleted + ``` diff --git a/terraform/files/kubernetes-manifests.d/cspo.yaml b/terraform/files/kubernetes-manifests.d/cspo.yaml index e39a806f..3e8e0a05 100644 --- a/terraform/files/kubernetes-manifests.d/cspo.yaml +++ b/terraform/files/kubernetes-manifests.d/cspo.yaml @@ -16,7 +16,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: openstackclusterstackreleases.infrastructure.clusterstack.x-k8s.io spec: group: infrastructure.clusterstack.x-k8s.io @@ -50,14 +50,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -73,14 +78,16 @@ spec: reconciling this cluster properties: kind: - description: Kind of the identity. Must be supported by the infrastructure + description: |- + Kind of the identity. Must be supported by the infrastructure provider and may be either cluster or namespace-scoped. minLength: 1 type: string name: - description: Name of the infrastructure identity to be used. Must - be either a cluster-scoped resource, or namespaced-scoped resource - the same namespace as the resource(s) being provisioned. + description: |- + Name of the infrastructure identity to be used. + Must be either a cluster-scoped resource, or namespaced-scoped + resource the same namespace as the resource(s) being provisioned. type: string required: - kind @@ -98,37 +105,37 @@ spec: operational state. properties: lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. format: date-time type: string message: - description: A human readable message indicating details about - the transition. This field may be empty. + description: |- + A human readable message indicating details about the transition. + This field may be empty. type: string reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. type: string severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. type: string status: description: Status of the condition, one of True, False, Unknown. type: string type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. type: string required: - lastTransitionTime @@ -150,7 +157,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: openstackclusterstackreleasetemplates.infrastructure.clusterstack.x-k8s.io spec: group: infrastructure.clusterstack.x-k8s.io @@ -170,14 +177,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -202,16 +214,16 @@ spec: used when reconciling this cluster properties: kind: - description: Kind of the identity. Must be supported by - the infrastructure provider and may be either cluster - or namespace-scoped. + description: |- + Kind of the identity. Must be supported by the infrastructure + provider and may be either cluster or namespace-scoped. minLength: 1 type: string name: - description: Name of the infrastructure identity to be - used. Must be either a cluster-scoped resource, or namespaced-scoped - resource the same namespace as the resource(s) being - provisioned. + description: |- + Name of the infrastructure identity to be used. + Must be either a cluster-scoped resource, or namespaced-scoped + resource the same namespace as the resource(s) being provisioned. type: string required: - kind @@ -238,7 +250,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: openstacknodeimagereleases.infrastructure.clusterstack.x-k8s.io spec: group: infrastructure.clusterstack.x-k8s.io @@ -272,14 +284,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -292,14 +309,16 @@ spec: reconciling this cluster properties: kind: - description: Kind of the identity. Must be supported by the infrastructure + description: |- + Kind of the identity. Must be supported by the infrastructure provider and may be either cluster or namespace-scoped. minLength: 1 type: string name: - description: Name of the infrastructure identity to be used. Must - be either a cluster-scoped resource, or namespaced-scoped resource - the same namespace as the resource(s) being provisioned. + description: |- + Name of the infrastructure identity to be used. + Must be either a cluster-scoped resource, or namespaced-scoped + resource the same namespace as the resource(s) being provisioned. type: string required: - kind @@ -312,11 +331,13 @@ spec: description: CreateOpts represents options used to create an image. properties: container_format: - description: ContainerFormat is the format of the container. - Valid values are ami, ari, aki, bare, and ovf. + description: |- + ContainerFormat is the format of the + container. Valid values are ami, ari, aki, bare, and ovf. type: string disk_format: - description: DiskFormat is the format of the disk. If set, + description: |- + DiskFormat is the format of the disk. If set, valid values are ami, ari, aki, vhd, vmdk, raw, qcow2, vdi, and iso. type: string @@ -324,12 +345,14 @@ spec: description: Id is the the image ID. type: string min_disk: - description: MinDisk is the amount of disk space in GB that - is required to boot the image. + description: |- + MinDisk is the amount of disk space in + GB that is required to boot the image. type: integer min_ram: - description: MinRAM is the amount of RAM in MB that is required - to boot the image. + description: |- + MinRAM is the amount of RAM in MB that + is required to boot the image. type: integer name: description: Name is the name of the new image. @@ -373,37 +396,37 @@ spec: operational state. properties: lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. format: date-time type: string message: - description: A human readable message indicating details about - the transition. This field may be empty. + description: |- + A human readable message indicating details about the transition. + This field may be empty. type: string reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. type: string severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. type: string status: description: Status of the condition, one of True, False, Unknown. type: string type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. type: string required: - lastTransitionTime @@ -743,7 +766,7 @@ spec: secretKeyRef: key: git-access-token name: cspo-cluster-stack-variables - image: ghcr.io/sovereigncloudstack/cspo:v0.1.0-alpha.3 + image: ghcr.io/sovereigncloudstack/cspo:v0.1.0-alpha.4 livenessProbe: httpGet: path: /healthz diff --git a/terraform/files/template/clusterctl.yaml.tmpl b/terraform/files/template/clusterctl.yaml.tmpl index 887f3285..8743ca33 100644 --- a/terraform/files/template/clusterctl.yaml.tmpl +++ b/terraform/files/template/clusterctl.yaml.tmpl @@ -38,8 +38,8 @@ DEPLOY_FLUX: ${deploy_flux} DEPLOY_METRICS: ${deploy_metrics} # OpenStack instance additional metadata -OPENSTACK_CONTROL_PLANE_MACHINE_METADATA: "{ %{ for metadata_key, metadata_value in controller_metadata ~} ${metadata_key}: '${metadata_value}', %{ endfor ~} }" -OPENSTACK_NODE_MACHINE_METADATA: "{ %{ for metadata_key, metadata_value in worker_metadata ~} ${metadata_key}: '${metadata_value}', %{ endfor ~} }" +OPENSTACK_CONTROL_PLANE_MACHINE_METADATA: "%{ if length(controller_metadata) > 0 }{ %{ for metadata_key, metadata_value in controller_metadata ~} ${metadata_key}: '${metadata_value}', %{ endfor ~} }%{ endif }" +OPENSTACK_NODE_MACHINE_METADATA: "%{ if length(worker_metadata) > 0 }{ %{ for metadata_key, metadata_value in worker_metadata ~} ${metadata_key}: '${metadata_value}', %{ endfor ~} }%{ endif }" # OpenStack flavors and machine count OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR: ${controller_flavor}