From 46e3006b1184ae4427f4fa15e7d0710bbb9b806e Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 14:00:38 +0000
Subject: [PATCH 01/14] Prepare KaaS v1 mgmt cluster to bootstrap v2.

This incluses setting the needed feature gate flags for CAPI plus a
deployment script for the CSO and CSPO pieces.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/bootstrap.sh         | 12 ++++++++-
 terraform/files/bin/install_cspo.sh      | 34 ++++++++++++++++++++++++
 terraform/files/bin/prepare_openstack.sh |  1 +
 3 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100755 terraform/files/bin/install_cspo.sh

diff --git a/terraform/files/bin/bootstrap.sh b/terraform/files/bin/bootstrap.sh
index 5826db2a..95260ff0 100755
--- a/terraform/files/bin/bootstrap.sh
+++ b/terraform/files/bin/bootstrap.sh
@@ -74,12 +74,22 @@ source ~/.kube-ps1/kube-ps1.sh
 PS1="\${PS1%\\\\\$ } \\\$(kube_ps1) [\\\$?]\\\$ "
 # We may do git commits and nano feels unusual ...
 export VISUAL=/usr/bin/vim
+
+# clusterctl beta features
+export CLUSTER_TOPOLOGY=true
+export EXP_CLUSTER_RESOURCE_SET=true
+
+# CSO settings
+export GIT_PROVIDER_B64=$(echo -n "github" | base64 -w0)
+export GIT_ORG_NAME_B64=$(echo -n "SovereignCloudStack" | base64 -w0)
+export GIT_REPOSITORY_NAME_B64=$(echo -n "cluster-stacks" | base64 -w0)
+# export GIT_ACCESS_TOKEN_B64=\$(echo -n "\$GIT_ACCESS_TOKEN" | base64 -w0)
 # eof
 EOF
 
 # openstack completion
 openstack complete > ~/.bash_openstack 2>/dev/null
-echo -e "#openstack completion\nsource ~/.bash_openstack" >> ~/.bash_aliases
+echo -e "#openstack complete\nsource ~/.bash_openstack" >> ~/.bash_aliases
 
 # set inputrc set tab once
 cat <<EOF > .inputrc
diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
new file mode 100755
index 00000000..85a7b688
--- /dev/null
+++ b/terraform/files/bin/install_cspo.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Bootstrap Cluster Stacks on a KaaS v1 management host
+# (c) Kurt Garloff <garloff@osb-alliance.com>, 3/2024
+# SPDX-License-Identifier: ASL-2.0
+cd 
+if test -e ~/.bash_aliases; then . ~/.bash_aliases; fi
+# Check out repos
+test_or_update()
+{
+	if test -d $1; then
+		cd $1
+		git update
+		cd
+	else
+		git clone https://github.com/SovereignCloudStack/$1
+	fi
+}	
+test_or_update cluster-stack-operator
+test_or_update cluster-stack-provider-openstack
+# envsubst helper (please always call with full path, as there is a name conflict)
+sudo apt-get install golang-go
+if test ! -x /usr/local/bin/envsubst; then
+	mkdir -p ~/tmp
+	GOBIN=~/tmp go install github.com/drone/envsubst/v2/cmd/envsubst@latest
+	sudo mv ~/tmp/envsubst /usr/local/bin/
+fi
+# Deploy CSO and CSPO
+if test -z "$GIT_ACCESS_TOKEN_B64"; then
+	echo "Please set GIT_ACCESS_TOKEN_B64 in your ~/.bash_aliases"
+	exit 1
+fi
+ENVSUBST=/usr/local/bin/envsubst
+#$ENVSUBST <
diff --git a/terraform/files/bin/prepare_openstack.sh b/terraform/files/bin/prepare_openstack.sh
index 3082e6eb..bda407c1 100755
--- a/terraform/files/bin/prepare_openstack.sh
+++ b/terraform/files/bin/prepare_openstack.sh
@@ -9,6 +9,7 @@ sudo patch -p2 -N -d /usr/lib/python3/dist-packages/keystoneauth1 < /tmp/fix-key
 
 # convenience
 echo "export OS_CLOUD=\"$OS_CLOUD\"" >> $HOME/.bash_aliases
+echo "export CLIFF_FIT_WIDTH=1" >> $HOME/.bash_aliases
 
 # Determine project ID and inject into cloud.conf
 PROJECTID=$(openstack application credential show "${PREFIX}-appcred" -f value -c project_id)

From 2bf8c9d0745aea88e8945ad26b8b77d7936a1c52 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 14:52:27 +0000
Subject: [PATCH 02/14] Complete CSPO bootstrap

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 164 +++++++++++++++++++++++++++-
 1 file changed, 158 insertions(+), 6 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 85a7b688..d41e9c8d 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -1,8 +1,16 @@
 #!/bin/bash
 #
 # Bootstrap Cluster Stacks on a KaaS v1 management host
+#
+# Usage: install_cspo.sh [csponame [clusterns]]
+# Defaults to cspo and cluster
+#
+# Needs OS_CLOUD to be set to your target cloud project
+#
 # (c) Kurt Garloff <garloff@osb-alliance.com>, 3/2024
 # SPDX-License-Identifier: ASL-2.0
+NAME=${1:-cspo}
+CLUSTER=${2:-cluster}
 cd 
 if test -e ~/.bash_aliases; then . ~/.bash_aliases; fi
 # Check out repos
@@ -19,16 +27,160 @@ test_or_update()
 test_or_update cluster-stack-operator
 test_or_update cluster-stack-provider-openstack
 # envsubst helper (please always call with full path, as there is a name conflict)
-sudo apt-get install golang-go
-if test ! -x /usr/local/bin/envsubst; then
+ENVSUBST=/usr/local/bin/envsubst
+if test ! -x $ENVSUBST; then
+	sudo apt-get install golang-go
 	mkdir -p ~/tmp
 	GOBIN=~/tmp go install github.com/drone/envsubst/v2/cmd/envsubst@latest
-	sudo mv ~/tmp/envsubst /usr/local/bin/
+	sudo mv ~/tmp/envsubst $ENVSUBST
 fi
 # Deploy CSO and CSPO
 if test -z "$GIT_ACCESS_TOKEN_B64"; then
-	echo "Please set GIT_ACCESS_TOKEN_B64 in your ~/.bash_aliases"
+	echo "Please set GIT_ACCESS_TOKEN_B64 in your ~/.bash_aliases" 1>&2
 	exit 1
 fi
-ENVSUBST=/usr/local/bin/envsubst
-#$ENVSUBST <
+# CSO manifests
+mkdir -p $NAME
+cd $NAME
+CSO_VERSION=$(curl https://api.github.com/repos/SovereignCloudStack/cluster-stack-operator/releases/latest -s | jq .name -r)
+curl -sSLO https://github.com/sovereignCloudStack/cluster-stack-operator/releases/download/${CSO_VERSION}/cso-infrastructure-components.yaml
+# CSPO manifests
+CSPO_VERSION=$(curl https://api.github.com/repos/SovereignCloudStack/cluster-stack-provider-openstack/releases/latest -s | jq .name -r)
+curl -sSLO https://github.com/sovereignCloudStack/cluster-stack-provider-openstack/releases/download/${CSPO_VERSION}/cspo-infrastructure-components.yaml
+$ENVSUBST < cso-infrastructure-components.yaml | kubectl apply -f -
+$ENVSUBST < cspo-infrastructure-components.yaml | kubectl apply -f -
+# Prepare for cluster templates
+kubectl create ns $CLUSTER
+# Create clouds.yaml (with app credential)
+if test ! -r clouds.yaml; then
+	#APPCREDS=$(openstack application credential list -f value -c ID -c Name -c "Project ID")
+	APPCRED=$(openstack application credential show $PREFIX-$NAME  >/dev/null)
+	if test $? = 0; then
+		echo "App Cred $PREFIX-CSPO exists, but no clouds.yaml, please delete it" 1>&2
+		exit 2
+	fi
+	# restricted AppCred should be OK, as we don't create dependant Sub-AppCreds, so no --unsrestricted
+	NEWCRED=$(openstack application credential create "$PREFIX-$NAME" --description "App Cred $PREFIX for $NAME" -f value -c id -c project_id -c secret)
+	if test $? != 0; then
+		echo "Application Credential generation failed." 1>&2
+		exit 2
+	fi
+	read APPCRED_ID APPCRED_PRJ APPCRED_SECRET < <(echo $NEWCRED)
+	echo "#Created AppCred $APPCRED_ID"
+	AUTH_URL=$(print-cloud.py | yq eval .clouds.${OS_CLOUD}.auth.auth_url -)
+	REGION=$(print-cloud.py | yq eval .clouds.${OS_CLOUD}.region_name -)
+	CACERT=$(print-cloud.py | yq eval '.clouds."'"$OS_CLOUD"'".cacert // "null"' -)
+	# In theory we could also make interface and id_api_vers variable,
+	# but let's do that once we find the necessity. Error handling makes
+	# it slightly complex, so it's not an obvious win.
+	cat >clouds.yaml <<EOT
+clouds:
+  #$PREFIX-$NAME:
+  openstack:
+    interface: public
+    identity_api_version: 3
+    region_name: $REGION
+    cacert: $CACERT
+    auth_type: "v3applicationcredential"
+    auth:
+      auth_url: $AUTH_URL
+      #project_id: $APPCRED_PRJ
+      application_credential_id: $APPCRED_ID
+      application_credential_secret: "$APPCRED_SECRET"
+EOT
+	if test "$CACERT" == "null"; then
+		sed -i '/    cacert:/d' clouds.yaml
+	fi
+	# And remove secret from env
+	unset APPCRED_SECRET NEWCRED
+fi
+# Create secret from clouds.yaml
+helm upgrade -i csp-helper https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz -f clouds.yaml
+# Store an example cluster-stack
+cat > clusterstack-alpha-1-28-v3.yaml <<EOT
+apiVersion: clusterstack.x-k8s.io/v1alpha1
+kind: ClusterStack
+metadata:
+  name: clusterstack
+  namespace: $CLUSTER
+spec:
+  provider: openstack
+  name: alpha
+  kubernetesVersion: "1.28"
+  channel: stable
+  autoSubscribe: false
+  providerRef:
+    apiVersion: infrastructure.clusterstack.x-k8s.io/v1alpha1
+    kind: OpenStackClusterStackReleaseTemplate
+    name: cspotemplate
+  versions:
+    - v3
+---
+apiVersion: infrastructure.clusterstack.x-k8s.io/v1alpha1
+kind: OpenStackClusterStackReleaseTemplate
+metadata:
+  name: cspotemplate
+  namespace: $CLUSTER
+spec:
+  template:
+    spec:
+      identityRef:
+        kind: Secret
+        name: openstack
+EOT
+cat >clusterresourceset-secret.yaml <<EOT
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+ name: crs-openstack-secret
+ namespace: $CLUSTER
+spec:
+ strategy: "Reconcile"
+ clusterSelector:
+   matchLabels:
+     managed-secret: cloud-config
+ resources:
+   - name: openstack-workload-cluster-secret
+     kind: Secret
+EOT
+cat >cluster-alpha-1-28-v3.yaml <<EOT
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: cs-$CLUSTER
+  namespace: $CLUSTER
+  labels:
+    managed-secret: cloud-config
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    serviceDomain: cluster.local
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+  topology:
+    variables:
+      - name: controller_flavor
+        value: "SCS-2V-4-50"
+      - name: worker_flavor
+        value: "SCS-2V-4-50"
+      - name: external_id
+        value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" # gx-scs
+    class: openstack-alpha-1-28-v3
+    controlPlane:
+      replicas: 1
+    version: v1.28.11
+    workers:
+      machineDeployments:
+        - class: capi-openstack-alpha-1-28
+          failureDomain: nova
+          name: capi-openstack-alpha-1-28
+          replicas: 3
+EOT
+#echo "Apply clusterstack-alpha-1-28-v3.yaml clusterresourceset-secret.yaml cluster-alpha-1-28-v3.yaml to create a workload cluster.
+echo "kubectl apply -f $NAME/clusterstack-alpha-1-28-v3.yaml"
+echo "kubectl apply -f $NAME/clusterresourceset-secret.yaml"
+# FIXME: Wait needed?
+echo "kubectl apply -f $NAME/cluster-alpha-1-28-v3.yaml"

From 9bb1393272bf8414f08e890f6b20f8b3569800bf Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:04:09 +0000
Subject: [PATCH 03/14] unset SSH_AUTH_SOCK before calling ssh or tofu

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/Makefile | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/terraform/Makefile b/terraform/Makefile
index 6e16d1fd..8053f359 100644
--- a/terraform/Makefile
+++ b/terraform/Makefile
@@ -88,6 +88,7 @@ create: init
 		echo "Management cluster with prefix $(PREFIX) already exists. This is not supported due to unexpected side-effects." && exit 1; \
 	else \
 		touch .deploy.$(ENVIRONMENT); \
+		unset SSH_AUTH_SOCK; \
 		tofu apply -auto-approve -var-file="environments/environment-$(ENVIRONMENT).tfvars" -var "git_reference=$(GITREFERENCE)" -var "git_repo=$(GITREPO)"; \
 	fi
 
@@ -96,6 +97,7 @@ show: init
 
 clean: init
 	@if test -e ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); then source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	ssh -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS "timeout 320 ~/bin/cleanup.sh || true"; fi
 	tofu destroy -auto-approve -var-file="environments/environment-$(ENVIRONMENT).tfvars" $(PARAMS)
 	@tofu workspace select default
@@ -140,12 +142,14 @@ list: init
 
 ssh: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVIRONMENT)
 	@source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	ssh -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS
 
 login: ssh
 
 get-kubeconfig: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVIRONMENT)
 	@source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	scp -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS:.kube/config .kubeconfig.$(ENVIRONMENT); \
 	scp -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS:$(TESTCLUSTER)/$(TESTCLUSTER).yaml $(TESTCLUSTER).yaml.$(ENVIRONMENT); \
 	scp -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS:$(TESTCLUSTER)/clusterctl.yaml clusterctl.$(TESTCLUSTER).yaml.$(ENVIRONMENT); \
@@ -155,6 +159,7 @@ get-kubeconfig: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVI
 
 deploy: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVIRONMENT)
 	@source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	ssh -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS "~/bin/deploy_cluster_api.sh"
 
 log: .deploy.$(ENVIRONMENT)
@@ -178,6 +183,7 @@ console: .deploy.$(ENVIRONMENT)
 
 check: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVIRONMENT)
 	@source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	ssh -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS "~/bin/sonobuoy.sh $(TESTCLUSTER) $(SONOMODE)"
 
 # Pass SONOMODE="--mode quick" for a quick check
@@ -204,6 +210,7 @@ watch: .deploy.id_rsa.$(ENVIRONMENT) .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT)
 	  YELLOW=$$(echo -e "\e[0;33m"); \
 	  NORM=$$(echo -e "\e[0;0m"); \
 	fi; \
+	unset SSH_AUTH_SOCK; \
 	while true; do \
 		LEN=$$(ssh -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS "sudo wc -l /var/log/cloud-init-output.log 2>/dev/null"); \
 		LEN=$${LEN%% *}; \
@@ -228,11 +235,13 @@ openstack: init
 
 k9s: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVIRONMENT)
 	@source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	ssh -t -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS \
 	"KUBECONFIG=/home/$(USERNAME)/.kube/config:/home/$(USERNAME)/$(TESTCLUSTER)/$(TESTCLUSTER).yaml k9s --all-namespaces"
 
 deploy-cso: .deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT) .deploy.id_rsa.$(ENVIRONMENT)
 	@source ./.deploy.MGMTCLUSTER_ADDRESS.$(ENVIRONMENT); \
+	unset SSH_AUTH_SOCK; \
 	ssh -t -o StrictHostKeyChecking=no -i .deploy.id_rsa.$(ENVIRONMENT) $(USERNAME)@$$MGMTCLUSTER_ADDRESS \
 	"KUBECONFIG=/home/$(USERNAME)/.kube/config kubectl apply -f /home/$(USERNAME)/kubernetes-manifests.d/cso.yaml --context kind-kind"
 

From 56a2bde6c555691bb48c69ce7be138d5520d0fb1 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:33:41 +0000
Subject: [PATCH 04/14] Fix git pull, import $PREFIX, rename namespace in helm

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index d41e9c8d..a15e3c5e 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -11,14 +11,15 @@
 # SPDX-License-Identifier: ASL-2.0
 NAME=${1:-cspo}
 CLUSTER=${2:-cluster}
-cd 
+cd
+. ~/.capi-settings
 if test -e ~/.bash_aliases; then . ~/.bash_aliases; fi
 # Check out repos
 test_or_update()
 {
 	if test -d $1; then
 		cd $1
-		git update
+		git pull
 		cd
 	else
 		git clone https://github.com/SovereignCloudStack/$1
@@ -29,7 +30,7 @@ test_or_update cluster-stack-provider-openstack
 # envsubst helper (please always call with full path, as there is a name conflict)
 ENVSUBST=/usr/local/bin/envsubst
 if test ! -x $ENVSUBST; then
-	sudo apt-get install golang-go
+	sudo apt-get -y install golang-go
 	mkdir -p ~/tmp
 	GOBIN=~/tmp go install github.com/drone/envsubst/v2/cmd/envsubst@latest
 	sudo mv ~/tmp/envsubst $ENVSUBST
@@ -95,7 +96,14 @@ EOT
 	unset APPCRED_SECRET NEWCRED
 fi
 # Create secret from clouds.yaml
-helm upgrade -i csp-helper https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz -f clouds.yaml
+curl -sSLO https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz
+tar xvzf csp-helper-chart.tgz
+# Replace namespace
+sed -i "/^{{\\- if include \"isAppCredential\" \\. \\-}}/{n
+i$CLUSTER
+d
+}" csp-helper-chart/templates/_helpers.tpl
+helm upgrade -i csp-helper csp-helper-chart -f clouds.yaml >/dev/null
 # Store an example cluster-stack
 cat > clusterstack-alpha-1-28-v3.yaml <<EOT
 apiVersion: clusterstack.x-k8s.io/v1alpha1

From abb41af88f888c692a78482688778e6d76aabe93 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:36:36 +0000
Subject: [PATCH 05/14] Per namespace filenames.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index a15e3c5e..5286642b 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -105,7 +105,7 @@ d
 }" csp-helper-chart/templates/_helpers.tpl
 helm upgrade -i csp-helper csp-helper-chart -f clouds.yaml >/dev/null
 # Store an example cluster-stack
-cat > clusterstack-alpha-1-28-v3.yaml <<EOT
+cat > clusterstack-alpha-1-28-v3-$CLUSTER.yaml <<EOT
 apiVersion: clusterstack.x-k8s.io/v1alpha1
 kind: ClusterStack
 metadata:
@@ -136,7 +136,7 @@ spec:
         kind: Secret
         name: openstack
 EOT
-cat >clusterresourceset-secret.yaml <<EOT
+cat >clusterresourceset-secret-$CLUSTER.yaml <<EOT
 apiVersion: addons.cluster.x-k8s.io/v1beta1
 kind: ClusterResourceSet
 metadata:
@@ -151,7 +151,7 @@ spec:
    - name: openstack-workload-cluster-secret
      kind: Secret
 EOT
-cat >cluster-alpha-1-28-v3.yaml <<EOT
+cat >cluster-alpha-1-28-v3-$CLUSTER.yaml <<EOT
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
@@ -187,8 +187,8 @@ spec:
           name: capi-openstack-alpha-1-28
           replicas: 3
 EOT
-#echo "Apply clusterstack-alpha-1-28-v3.yaml clusterresourceset-secret.yaml cluster-alpha-1-28-v3.yaml to create a workload cluster.
-echo "kubectl apply -f $NAME/clusterstack-alpha-1-28-v3.yaml"
-echo "kubectl apply -f $NAME/clusterresourceset-secret.yaml"
+echo "# Perform these to create a workload cluster ..."
+echo "kubectl apply -f $NAME/clusterstack-alpha-1-28-v3-$CLUSTER.yaml"
+echo "kubectl apply -f $NAME/clusterresourceset-secret-$CLUSTER.yaml"
 # FIXME: Wait needed?
-echo "kubectl apply -f $NAME/cluster-alpha-1-28-v3.yaml"
+echo "kubectl apply -f $NAME/cluster-alpha-1-28-v3-$CLUSTER.yaml"

From 23d0918fef25dfdd4af311565f8957d2e7ebb4d2 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:42:32 +0000
Subject: [PATCH 06/14] Cosmetics.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 5286642b..8c99a480 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -25,6 +25,7 @@ test_or_update()
 		git clone https://github.com/SovereignCloudStack/$1
 	fi
 }	
+test_or_update cluster-stacks
 test_or_update cluster-stack-operator
 test_or_update cluster-stack-provider-openstack
 # envsubst helper (please always call with full path, as there is a name conflict)
@@ -105,6 +106,8 @@ d
 }" csp-helper-chart/templates/_helpers.tpl
 helm upgrade -i csp-helper csp-helper-chart -f clouds.yaml >/dev/null
 # Store an example cluster-stack
+# Note: These should preferably be taken from the checked out repos.
+# Currently, we use the content from https://input.scs.community/_HeOTRCRSu2Uf2SfMSoOkQ?both#
 cat > clusterstack-alpha-1-28-v3-$CLUSTER.yaml <<EOT
 apiVersion: clusterstack.x-k8s.io/v1alpha1
 kind: ClusterStack
@@ -179,7 +182,7 @@ spec:
     class: openstack-alpha-1-28-v3
     controlPlane:
       replicas: 1
-    version: v1.28.11
+    version: v1.28.7
     workers:
       machineDeployments:
         - class: capi-openstack-alpha-1-28

From 62c56c332bee5d227cb00eea6c488cde18d26e85 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:47:14 +0000
Subject: [PATCH 07/14] Error message on missing GIT_* environment.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 8c99a480..30f34846 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -37,6 +37,10 @@ if test ! -x $ENVSUBST; then
 	sudo mv ~/tmp/envsubst $ENVSUBST
 fi
 # Deploy CSO and CSPO
+if test -z "$GIT_PROVIDER_B64"; then
+	echo "Please add GIT_[PROVIDER|ORG_NAME_REPOSITORY_NAME|ACCESS_TOKEN]_B64 to ~/.bash_aliases, see bootstrap.sh" 1>&2
+	exit 1
+fi
 if test -z "$GIT_ACCESS_TOKEN_B64"; then
 	echo "Please set GIT_ACCESS_TOKEN_B64 in your ~/.bash_aliases" 1>&2
 	exit 1

From 431740fdde0e1121261fac16fd8cc93591c7962a Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:50:53 +0000
Subject: [PATCH 08/14] Comment out namespace creation. csp-helper does it.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 30f34846..0999ce3d 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -56,7 +56,7 @@ curl -sSLO https://github.com/sovereignCloudStack/cluster-stack-provider-opensta
 $ENVSUBST < cso-infrastructure-components.yaml | kubectl apply -f -
 $ENVSUBST < cspo-infrastructure-components.yaml | kubectl apply -f -
 # Prepare for cluster templates
-kubectl create ns $CLUSTER
+# kubectl create ns $CLUSTER	# Not needed, helm csp-helper does it
 # Create clouds.yaml (with app credential)
 if test ! -r clouds.yaml; then
 	#APPCREDS=$(openstack application credential list -f value -c ID -c Name -c "Project ID")

From 5da0e293d613226a57ba708b60433d22ecd1db51 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 15:53:06 +0000
Subject: [PATCH 09/14] Silence tar.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 0999ce3d..7fd95556 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -102,7 +102,7 @@ EOT
 fi
 # Create secret from clouds.yaml
 curl -sSLO https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz
-tar xvzf csp-helper-chart.tgz
+tar xzf csp-helper-chart.tgz
 # Replace namespace
 sed -i "/^{{\\- if include \"isAppCredential\" \\. \\-}}/{n
 i$CLUSTER

From e44f8844c7e0ae6ae1de6b0528654f082c91f45c Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 16:02:40 +0000
Subject: [PATCH 10/14] Use auto-detected external network

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 7fd95556..9d558101 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -46,6 +46,7 @@ if test -z "$GIT_ACCESS_TOKEN_B64"; then
 	exit 1
 fi
 # CSO manifests
+EXTID=$(openstack network list --external -f value -c ID | head -n1)
 mkdir -p $NAME
 cd $NAME
 CSO_VERSION=$(curl https://api.github.com/repos/SovereignCloudStack/cluster-stack-operator/releases/latest -s | jq .name -r)
@@ -182,7 +183,7 @@ spec:
       - name: worker_flavor
         value: "SCS-2V-4-50"
       - name: external_id
-        value: "ebfe5546-f09f-4f42-ab54-094e457d42ec" # gx-scs
+        value: "$EXTID"
     class: openstack-alpha-1-28-v3
     controlPlane:
       replicas: 1

From 4da0b68f4a56c9ddb46a1b606b2fe61c4ede9316 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 17:31:12 +0000
Subject: [PATCH 11/14] Set cluster version back to 1.28.6 to match k8s.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 9d558101..1b7b5c25 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -101,9 +101,9 @@ EOT
 	# And remove secret from env
 	unset APPCRED_SECRET NEWCRED
 fi
+# export OS_CLOUD=openstack
 # Create secret from clouds.yaml
-curl -sSLO https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz
-tar xzf csp-helper-chart.tgz
+curl -sSL https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz | tar xv
 # Replace namespace
 sed -i "/^{{\\- if include \"isAppCredential\" \\. \\-}}/{n
 i$CLUSTER
@@ -187,7 +187,7 @@ spec:
     class: openstack-alpha-1-28-v3
     controlPlane:
       replicas: 1
-    version: v1.28.7
+    version: v1.28.6
     workers:
       machineDeployments:
         - class: capi-openstack-alpha-1-28

From 3cbb4fe89958b0009a73e0f750f2b38cd469b90b Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 19:02:28 +0000
Subject: [PATCH 12/14] Fix multiple namespaces. Deploy Secret.

So we deploy helm into a namespace and avoid helm managing the
namespace. This would otherwise result in helm deleting the
old namespace and the CRS when we want to use another namespace.
This would leave CAPI in a really bad state, unable to talk to
the infra, unable to clean up.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 1b7b5c25..3874110a 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -57,7 +57,6 @@ curl -sSLO https://github.com/sovereignCloudStack/cluster-stack-provider-opensta
 $ENVSUBST < cso-infrastructure-components.yaml | kubectl apply -f -
 $ENVSUBST < cspo-infrastructure-components.yaml | kubectl apply -f -
 # Prepare for cluster templates
-# kubectl create ns $CLUSTER	# Not needed, helm csp-helper does it
 # Create clouds.yaml (with app credential)
 if test ! -r clouds.yaml; then
 	#APPCREDS=$(openstack application credential list -f value -c ID -c Name -c "Project ID")
@@ -109,7 +108,9 @@ sed -i "/^{{\\- if include \"isAppCredential\" \\. \\-}}/{n
 i$CLUSTER
 d
 }" csp-helper-chart/templates/_helpers.tpl
-helm upgrade -i csp-helper csp-helper-chart -f clouds.yaml >/dev/null
+# kubectl create ns $CLUSTER	# Not needed, helm csp-helper does it
+rm -f csp-helper-chart/templates/namespace.yaml
+helm upgrade --create-namespace -n $CLUSTER -i csp-helper csp-helper-chart -f clouds.yaml >/dev/null
 # Store an example cluster-stack
 # Note: These should preferably be taken from the checked out repos.
 # Currently, we use the content from https://input.scs.community/_HeOTRCRSu2Uf2SfMSoOkQ?both#
@@ -195,8 +196,11 @@ spec:
           name: capi-openstack-alpha-1-28
           replicas: 3
 EOT
-echo "# Perform these to create a workload cluster ..."
-echo "kubectl apply -f $NAME/clusterstack-alpha-1-28-v3-$CLUSTER.yaml"
-echo "kubectl apply -f $NAME/clusterresourceset-secret-$CLUSTER.yaml"
-# FIXME: Wait needed?
-echo "kubectl apply -f $NAME/cluster-alpha-1-28-v3-$CLUSTER.yaml"
+kubectl apply -f clusterresourceset-secret-$CLUSTER.yaml
+echo "# Perform these to create a workload cluster (after editing as desired) ..."
+echo "kubectl apply -f ~/$NAME/clusterstack-alpha-1-28-v3-$CLUSTER.yaml"
+echo "kubectl apply -f ~/$NAME/cluster-alpha-1-28-v3-$CLUSTER.yaml"
+# FIXME: Code from create_cluster.sh would help here ...
+echo "# Wait for cluster to be ready ..."
+echo "clusterctl -n $CLUSTER get kubeconfig cs-$CLUSTER > ~/$NAME/cs-$CLUSTER.yaml"
+

From 72e2f19044a8e7f9ed4db02ec3b383634502a5b8 Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Sat, 9 Mar 2024 20:07:21 +0000
Subject: [PATCH 13/14] Streamline flavors used in example.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 3874110a..2488705c 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -180,9 +180,9 @@ spec:
   topology:
     variables:
       - name: controller_flavor
-        value: "SCS-2V-4-50"
+        value: "SCS-2V-4-20"
       - name: worker_flavor
-        value: "SCS-2V-4-50"
+        value: "SCS-2V-8-20"
       - name: external_id
         value: "$EXTID"
     class: openstack-alpha-1-28-v3

From e7da6ac058e94f44bb00fa47e34f483504d5030f Mon Sep 17 00:00:00 2001
From: Kurt Garloff <kurt@garloff.de>
Date: Tue, 11 Jun 2024 06:33:21 +0000
Subject: [PATCH 14/14] Adjust to latest openstack-csp-helper.

Signed-off-by: Kurt Garloff <kurt@garloff.de>
---
 terraform/files/bin/install_cspo.sh | 35 ++++++++++++++++-------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/terraform/files/bin/install_cspo.sh b/terraform/files/bin/install_cspo.sh
index 2488705c..81e23599 100755
--- a/terraform/files/bin/install_cspo.sh
+++ b/terraform/files/bin/install_cspo.sh
@@ -100,21 +100,23 @@ EOT
 	# And remove secret from env
 	unset APPCRED_SECRET NEWCRED
 fi
+chmod 0640 clouds.yaml
 # export OS_CLOUD=openstack
 # Create secret from clouds.yaml
-curl -sSL https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz | tar xv
+#curl -sSL https://github.com/SovereignCloudStack/cluster-stacks/releases/download/openstack-alpha-1-28-v3/csp-helper-chart.tgz | tar xv
+#rm -f openstack-csp-helper/templates/namespace.yaml
+curl -sSL https://github.com/SovereignCloudStack/openstack-csp-helper/releases/download/latest/openstack-csp-helper.tgz | tar xv
 # Replace namespace
 sed -i "/^{{\\- if include \"isAppCredential\" \\. \\-}}/{n
 i$CLUSTER
 d
-}" csp-helper-chart/templates/_helpers.tpl
+}" openstack-csp-helper/templates/_helpers.tpl
 # kubectl create ns $CLUSTER	# Not needed, helm csp-helper does it
-rm -f csp-helper-chart/templates/namespace.yaml
-helm upgrade --create-namespace -n $CLUSTER -i csp-helper csp-helper-chart -f clouds.yaml >/dev/null
+helm upgrade --create-namespace -n $CLUSTER -i $CLUSTER-credentials openstack-csp-helper -f clouds.yaml >/dev/null
 # Store an example cluster-stack
 # Note: These should preferably be taken from the checked out repos.
 # Currently, we use the content from https://input.scs.community/_HeOTRCRSu2Uf2SfMSoOkQ?both#
-cat > clusterstack-alpha-1-28-v3-$CLUSTER.yaml <<EOT
+cat > clusterstack-alpha-1-29-v3-$CLUSTER.yaml <<EOT
 apiVersion: clusterstack.x-k8s.io/v1alpha1
 kind: ClusterStack
 metadata:
@@ -123,7 +125,7 @@ metadata:
 spec:
   provider: openstack
   name: alpha
-  kubernetesVersion: "1.28"
+  kubernetesVersion: "1.29"
   channel: stable
   autoSubscribe: false
   providerRef:
@@ -145,6 +147,7 @@ spec:
         kind: Secret
         name: openstack
 EOT
+# No longer needed (part of openstack-csp-helper now)
 cat >clusterresourceset-secret-$CLUSTER.yaml <<EOT
 apiVersion: addons.cluster.x-k8s.io/v1beta1
 kind: ClusterResourceSet
@@ -160,7 +163,7 @@ spec:
    - name: openstack-workload-cluster-secret
      kind: Secret
 EOT
-cat >cluster-alpha-1-28-v3-$CLUSTER.yaml <<EOT
+cat >cluster-alpha-1-29-v3-$CLUSTER.yaml <<EOT
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
@@ -180,26 +183,26 @@ spec:
   topology:
     variables:
       - name: controller_flavor
-        value: "SCS-2V-4-20"
+        value: "SCS-2V-4-20s"
       - name: worker_flavor
-        value: "SCS-2V-8-20"
+        value: "SCS-2V-8-50"
       - name: external_id
         value: "$EXTID"
-    class: openstack-alpha-1-28-v3
+    class: openstack-alpha-1-29-v3
     controlPlane:
       replicas: 1
-    version: v1.28.6
+    version: v1.29.3
     workers:
       machineDeployments:
-        - class: capi-openstack-alpha-1-28
+        - class: openstack-alpha-1-29-v3
           failureDomain: nova
-          name: capi-openstack-alpha-1-28
+          name: openstack-alpha-1-29-v3
           replicas: 3
 EOT
-kubectl apply -f clusterresourceset-secret-$CLUSTER.yaml
+#kubectl apply -f clusterresourceset-secret-$CLUSTER.yaml
 echo "# Perform these to create a workload cluster (after editing as desired) ..."
-echo "kubectl apply -f ~/$NAME/clusterstack-alpha-1-28-v3-$CLUSTER.yaml"
-echo "kubectl apply -f ~/$NAME/cluster-alpha-1-28-v3-$CLUSTER.yaml"
+echo "kubectl apply -f ~/$NAME/clusterstack-alpha-1-29-v3-$CLUSTER.yaml"
+echo "kubectl apply -f ~/$NAME/cluster-alpha-1-29-v3-$CLUSTER.yaml"
 # FIXME: Code from create_cluster.sh would help here ...
 echo "# Wait for cluster to be ready ..."
 echo "clusterctl -n $CLUSTER get kubeconfig cs-$CLUSTER > ~/$NAME/cs-$CLUSTER.yaml"