ci: Add workload cluster creation and kubectl verification to smoke test

This ensures the Cluster API Operator can successfully create and manage
a functional Kubernetes cluster where kubectl commands work properly.

Signed-off-by: kahirokunn <[email protected]>

Author: kahirokunn

.github/workflows/smoke-test.yaml

@@ -33,6 +33,12 @@ jobs:
           chmod +x kubectl
           sudo mv kubectl /usr/local/bin/
 
+      - name: Install yq
+        run: |
+          wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O yq
+          chmod +x yq
+          sudo mv yq /usr/local/bin/
+
       - name: Install Helm
         run: |
           curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
@@ -59,7 +65,21 @@ jobs:
         run: |
           chmod +x ./hack/ensure-kind.sh
           ./hack/ensure-kind.sh
-          kind create cluster --name capi-operator-smoke-test --wait 5m
+
+          # Create kind cluster with Docker socket mount for CAPD
+          cat <<EOF > /tmp/kind-config.yaml
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          nodes:
+          - role: control-plane
+            extraMounts:
+            - hostPath: /var/run/docker.sock
+              containerPath: /var/run/docker.sock
+            - hostPath: /var/lib/docker
+              containerPath: /var/lib/docker
+          EOF
+
+          kind create cluster --name capi-operator-smoke-test --config /tmp/kind-config.yaml --wait 5m
           kubectl cluster-info --context kind-capi-operator-smoke-test
 
       - name: Load Docker image to kind
@@ -115,6 +135,12 @@ jobs:
           core:
             cluster-api:
               namespace: capi-system
+          bootstrap:
+            kubeadm:
+              namespace: capi-kubeadm-bootstrap-system
+          controlPlane:
+            kubeadm:
+              namespace: capi-kubeadm-control-plane-system
           infrastructure:
             docker:
               namespace: capd-system
@@ -133,20 +159,39 @@ jobs:
 
           helm install capi-providers "$PROVIDERS_CHART_PACKAGE" \
             -f /tmp/providers-values.yaml \
-            --wait
+            --wait \
+            --timeout 3m
 
       - name: Wait for providers to be ready
         run: |
-          echo "Waiting for Core Provider to be ready..."
+          echo "=== Waiting for Core Provider to be ready ==="
           kubectl wait --for=condition=Ready --timeout=300s -n capi-system coreprovider/cluster-api || true
 
-          echo "Waiting for Infrastructure Provider to be ready..."
+          # Additional check for CAPD provider to ensure Docker socket is accessible
+          echo -e "\n=== Checking Docker socket access in CAPD pod ==="
+          kubectl exec -n capd-system deployment/capd-controller-manager -- ls -la /var/run/docker.sock || echo "Docker socket not mounted"
+
+          echo -e "\n=== Waiting for Bootstrap Provider to be ready ==="
+          kubectl wait --for=condition=Ready --timeout=300s -n capi-kubeadm-bootstrap-system bootstrapprovider/kubeadm || true
+
+          echo -e "\n=== Waiting for Control Plane Provider to be ready ==="
+          kubectl wait --for=condition=Ready --timeout=300s -n capi-kubeadm-control-plane-system controlplaneprovider/kubeadm || true
+
+          echo -e "\n=== Waiting for Infrastructure Provider to be ready ==="
           kubectl wait --for=condition=Ready --timeout=300s -n capd-system infrastructureprovider/docker || true
 
           # Additional wait for deployments
+          echo -e "\n=== Waiting for provider deployments ==="
           kubectl wait --for=condition=Available --timeout=300s -n capi-system deployment/capi-controller-manager || true
+          kubectl wait --for=condition=Available --timeout=300s -n capi-kubeadm-bootstrap-system deployment/capi-kubeadm-bootstrap-controller-manager || true
+          kubectl wait --for=condition=Available --timeout=300s -n capi-kubeadm-control-plane-system deployment/capi-kubeadm-control-plane-controller-manager || true
           kubectl wait --for=condition=Available --timeout=300s -n capd-system deployment/capd-controller-manager || true
 
+          # Wait for webhooks to be ready
+          echo -e "\n=== Waiting for webhook services ==="
+          kubectl wait --for=jsonpath='{.status.loadBalancer}' --timeout=300s -n capi-kubeadm-bootstrap-system service/capi-kubeadm-bootstrap-webhook-service || true
+          kubectl wait --for=jsonpath='{.status.loadBalancer}' --timeout=300s -n capi-kubeadm-control-plane-system service/capi-kubeadm-control-plane-webhook-service || true
+
       - name: Verify installation
         run: |
           echo "=== Cluster API Operator Status ==="
@@ -156,13 +201,27 @@ jobs:
           kubectl get coreprovider -A -o wide
           kubectl describe coreprovider -n capi-system cluster-api || true
 
+          echo -e "\n=== Bootstrap Provider Status ==="
+          kubectl get bootstrapprovider -A -o wide
+          kubectl describe bootstrapprovider -n capi-kubeadm-bootstrap-system kubeadm || true
+
+          echo -e "\n=== Control Plane Provider Status ==="
+          kubectl get controlplaneprovider -A -o wide
+          kubectl describe controlplaneprovider -n capi-kubeadm-control-plane-system kubeadm || true
+
           echo -e "\n=== Infrastructure Provider Status ==="
           kubectl get infrastructureprovider -A -o wide
           kubectl describe infrastructureprovider -n capd-system docker || true
 
           echo -e "\n=== All Pods ==="
           kubectl get pods -A | grep -E "(capi-|capd-)"
 
+          echo -e "\n=== Webhook Services ==="
+          kubectl get svc -A | grep webhook
+
+          echo -e "\n=== Webhook Certificates ==="
+          kubectl get certificate,certificaterequest -A | grep -E "(capi-|capd-)"
+
           echo -e "\n=== CRDs ==="
           kubectl get crds | grep -E "(cluster.x-k8s.io|operator.cluster.x-k8s.io)"
 
@@ -176,6 +235,22 @@ jobs:
             exit 1
           fi
 
+          # Check if bootstrap provider is ready
+          BOOTSTRAP_READY=$(kubectl get bootstrapprovider -n capi-kubeadm-bootstrap-system kubeadm -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
+          if [ "$BOOTSTRAP_READY" != "True" ]; then
+            echo "Bootstrap provider is not ready"
+            kubectl get bootstrapprovider -n capi-kubeadm-bootstrap-system kubeadm -o yaml
+            exit 1
+          fi
+
+          # Check if control plane provider is ready
+          CONTROLPLANE_READY=$(kubectl get controlplaneprovider -n capi-kubeadm-control-plane-system kubeadm -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
+          if [ "$CONTROLPLANE_READY" != "True" ]; then
+            echo "Control plane provider is not ready"
+            kubectl get controlplaneprovider -n capi-kubeadm-control-plane-system kubeadm -o yaml
+            exit 1
+          fi
+
           # Check if infrastructure provider is ready
           INFRA_READY=$(kubectl get infrastructureprovider -n capd-system docker -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
           if [ "$INFRA_READY" != "True" ]; then
@@ -186,6 +261,120 @@ jobs:
 
           echo "All providers are ready!"
 
+          # Additional webhook readiness check
+          echo -e "\n=== Checking webhook endpoints ==="
+          kubectl get endpoints -A | grep webhook
+
+      - name: Download cluster manifest
+        run: |
+          echo "=== Downloading cluster manifest ==="
+          curl -L https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/refs/heads/main/test/infrastructure/docker/examples/simple-cluster.yaml -o simple-cluster.yaml
+
+          # Show the manifest for debugging
+          echo "=== Cluster manifest ==="
+          cat simple-cluster.yaml
+
+          # Extract cluster name from the manifest using yq
+          CLUSTER_NAME=$(yq eval 'select(.kind == "Cluster") | .metadata.name' simple-cluster.yaml)
+
+          # Ensure cluster name was extracted successfully
+          if [ -z "$CLUSTER_NAME" ]; then
+            echo "ERROR: Failed to extract cluster name from simple-cluster.yaml"
+            echo "Please check the manifest structure"
+            exit 1
+          fi
+
+          echo "Detected cluster name: $CLUSTER_NAME"
+          echo "CLUSTER_NAME=$CLUSTER_NAME" >> $GITHUB_ENV
+
+      - name: Create workload cluster
+        run: |
+          echo "=== Pre-creation diagnostics ==="
+          echo "Checking webhook services..."
+          kubectl get svc -A | grep webhook
+
+          echo -e "\nChecking webhook endpoints..."
+          kubectl get endpoints -A | grep webhook
+
+          echo -e "\nChecking webhook certificates..."
+          kubectl get secret -A | grep webhook-service-cert
+
+          echo -e "\n=== Creating workload cluster ==="
+          kubectl apply -f simple-cluster.yaml
+
+          echo -e "\n=== Cluster resources created ==="
+          kubectl get cluster,dockercluster,kubeadmcontrolplane,machinedeployment -A
+
+      - name: Wait for cluster to be ready
+        run: |
+          echo "=== Waiting for cluster to be provisioned ==="
+          kubectl wait --for=condition=Ready --timeout=600s cluster/${CLUSTER_NAME}
+
+          echo "=== Waiting for control plane to be initialized ==="
+          kubectl wait --for=condition=Ready --timeout=600s kubeadmcontrolplane -l cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}
+
+          echo "=== Waiting for first control plane node ==="
+          kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 --timeout=600s kubeadmcontrolplane -l cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}
+
+          echo "=== Cluster status ==="
+          kubectl get cluster ${CLUSTER_NAME} -o wide
+          kubectl get machines -l cluster.x-k8s.io/cluster-name=${CLUSTER_NAME}
+
+      - name: Get workload cluster kubeconfig
+        run: |
+          echo "=== Getting workload cluster kubeconfig ==="
+          # Get kubeconfig from the cluster
+          kubectl get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath='{.data.value}' | base64 -d > ${CLUSTER_NAME}.kubeconfig
+
+          echo "=== Testing kubeconfig ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig cluster-info || echo "Cluster may not be ready yet"
+
+      - name: Verify kubectl commands work on workload cluster
+        run: |
+          echo "=== Testing kubectl get po on workload cluster ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get po -A
+
+          echo -e "\n=== Testing kubectl get nodes ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get nodes
+
+          echo -e "\n=== Waiting for system pods to be ready ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig wait --for=condition=Ready --timeout=300s pods -n kube-system -l k8s-app=kube-proxy
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig wait --for=condition=Ready --timeout=300s pods -n kube-system -l component=kube-apiserver
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig wait --for=condition=Ready --timeout=300s pods -n kube-system -l component=kube-controller-manager
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig wait --for=condition=Ready --timeout=300s pods -n kube-system -l component=kube-scheduler
+
+      - name: Deploy and test sample application
+        run: |
+          echo "=== Deploying nginx test application ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig create deployment nginx --image=nginx:alpine --replicas=2
+
+          echo "=== Waiting for deployment to be ready ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig wait --for=condition=Available --timeout=120s deployment/nginx
+
+          echo "=== Verifying pods are running ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get po -l app=nginx
+
+          echo "=== Creating a service ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig expose deployment nginx --port=80 --type=ClusterIP
+
+          echo "=== Verifying service ==="
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get svc nginx
+
+      - name: Verify cluster functionality
+        run: |
+          echo "=== Final cluster verification ==="
+          echo "Cluster nodes:"
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get nodes -o wide
+
+          echo -e "\nAll pods:"
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get po -A
+
+          echo -e "\nAll services:"
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig get svc -A
+
+          echo -e "\nCluster info:"
+          kubectl --kubeconfig=${CLUSTER_NAME}.kubeconfig cluster-info
+
       - name: Collect debug information on failure
         if: failure()
         run: |
@@ -198,9 +387,36 @@ jobs:
           echo -e "\n=== Core Provider Logs ==="
           kubectl logs -n capi-system deployment/capi-controller-manager --tail=100 || true
 
+          echo -e "\n=== Bootstrap Provider Logs ==="
+          kubectl logs -n capi-kubeadm-bootstrap-system deployment/capi-kubeadm-bootstrap-controller-manager --tail=100 || true
+
+          echo -e "\n=== Control Plane Provider Logs ==="
+          kubectl logs -n capi-kubeadm-control-plane-system deployment/capi-kubeadm-control-plane-controller-manager --tail=100 || true
+
           echo -e "\n=== Infrastructure Provider Logs ==="
           kubectl logs -n capd-system deployment/capd-controller-manager --tail=100 || true
 
+          echo -e "\n=== Webhook Services and Endpoints ==="
+          kubectl get svc,endpoints -A | grep webhook || true
+
+          echo -e "\n=== Webhook Certificates ==="
+          kubectl get certificate,certificaterequest,secret -A | grep -E "(webhook|serving-cert)" || true
+
+          echo -e "\n=== Cluster Resources ==="
+          kubectl get cluster,dockercluster,kubeadmcontrolplane,machine,dockermachine -A -o wide || true
+
+          echo -e "\n=== Describe Cluster ==="
+          kubectl describe cluster ${CLUSTER_NAME} || true
+
+          echo -e "\n=== Describe Machines ==="
+          kubectl describe machines -l cluster.x-k8s.io/cluster-name=${CLUSTER_NAME} || true
+
+          echo -e "\n=== Docker Containers ==="
+          docker ps -a | grep -E "(smoke-test|kind)" || true
+
+          echo -e "\n=== Kind Clusters ==="
+          kind get clusters || true
+
           echo -e "\n=== Describe Failed Pods ==="
           kubectl get pods -A | grep -v Running | grep -v Completed | tail -n +2 | while read namespace name ready status restarts age; do
             echo "Describing pod $name in namespace $namespace"
@@ -211,4 +427,19 @@ jobs:
       - name: Clean up
         if: always()
         run: |
+          echo "=== Cleaning up kind clusters ==="
+          # List all kind clusters before cleanup
+          echo "Current kind clusters:"
+          kind get clusters || true
+
+          # Delete workload cluster if it exists
+          echo "Deleting workload cluster: ${CLUSTER_NAME}"
+          kind delete cluster --name ${CLUSTER_NAME} || true
+
+          # Delete management cluster
+          echo "Deleting management cluster: capi-operator-smoke-test"
           kind delete cluster --name capi-operator-smoke-test || true
+
+          # Verify all clusters are deleted
+          echo "Remaining kind clusters:"
+          kind get clusters || true