Fix Argo Deletion Again

According to argoproj/argo-cd#12376 waves
don't actually work properly for deletion.  I can see in the code it
does a reverse sort and picks off the largest wave first, but I guess
there's a bug where it'll kick off the next wave before the first has
actually deleted (or something to that effect). However, we can tell
argo not to delete something and let CAPI do its own thing, which
appears to work more reliably. Anecdotally, the secret at wave -1 still
gets deleted last, which defies all logic and tells some something kinda
works in some situations and not others!!

charts/cluster-api-cluster-openstack/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: cluster-api-cluster-openstack
 description: A Helm chart to deploy a Kubernetes Cluster
 type: application
-version: v0.3.14
+version: v0.3.15
 icon: https://raw.githubusercontent.com/eschercloudai/helm-cluster-api/main/icons/default.png

charts/cluster-api-cluster-openstack/templates/cluster.yaml

@@ -5,13 +5,6 @@ metadata:
   name: {{ .Release.Name }}
   labels:
     {{- include "openstackcluster.labels" . | nindent 4 }}
-  annotations:
-    # If we just delete all the things, I believe there is a race where
-    # CAPO blows away the cluster infrastructure before it has a chance
-    # to delete the machines, and we get into a deadlock state.  By
-    # provisioning the cluster last, and deleting it first, we should be
-    # able to let CAPI take care of its own ordering.
-    argocd.argoproj.io/sync-wave: "1"
 spec:
   clusterNetwork:
     pods:
@@ -40,6 +33,9 @@ metadata:
   name: {{ .Release.Name }}
   labels:
     {{- include "openstackcluster.labels" . | nindent 4 }}
+  annotations:
+    # Let CAPO do this in its chosen order.
+    argocd.argoproj.io/sync-options: Delete=false
 spec:
   cloudName: {{ .Values.openstack.cloud }}
   identityRef:

charts/cluster-api-cluster-openstack/templates/control-plane.yaml

@@ -5,6 +5,9 @@ metadata:
   name: "{{ .Release.Name }}-control-plane"
   labels:
     {{- include "openstackcluster.labels" . | nindent 4 }}
+  annotations:
+    # Let CAPO do this in its chosen order.
+    argocd.argoproj.io/sync-options: Delete=false
 spec:
   version: "{{ .Values.controlPlane.version }}"
   replicas: {{ .Values.controlPlane.replicas }}
@@ -62,6 +65,9 @@ metadata:
   name: {{ .Release.Name }}-control-plane-{{ include "openstack.discriminator.control-plane" . }}
   labels:
     {{- include "openstackcluster.labels" . | nindent 4 }}
+  annotations:
+    # Let CAPO do this in its chosen order.
+    argocd.argoproj.io/sync-options: Delete=false
 spec:
   template:
     spec:

charts/cluster-api-cluster-openstack/templates/workload.yaml

@@ -20,10 +20,8 @@ metadata:
   labels:
     {{- include "openstackcluster.labels" $ | nindent 4 }}
   annotations:
-    # See comment in cluster.yaml.  ArgoCD has some smarts that monitor
-    # the status of Cluster/Machine/MachineDeployment, so we need to
-    # provision this after any referenced templates.
-    argocd.argoproj.io/sync-wave: "2"
+    # Let CAPO do this in its chosen order.
+    argocd.argoproj.io/sync-options: Delete=false
     {{- include "openstackcluster.autoscalingAnnotations" $pool | nindent 4 }}
 spec:
   clusterName: "{{ $.Release.Name }}"
@@ -53,6 +51,9 @@ metadata:
   name: {{ $pool_name_discriminated }}
   labels:
     {{- include "openstackcluster.labels" $ | nindent 4 }}
+  annotations:
+    # Let CAPO do this in its chosen order.
+    argocd.argoproj.io/sync-options: Delete=false
 spec:
   template:
     spec:
@@ -77,6 +78,9 @@ metadata:
   name: {{ $pool_name_discriminated }}
   labels:
     {{- include "openstackcluster.labels" $ | nindent 4 }}
+  annotations:
+    # Let CAPO do this in its chosen order.
+    argocd.argoproj.io/sync-options: Delete=false
 spec:
   template:
     spec: