fix aks post deployment bugs

infra/azure/terraform/k8s-infra-prow-build/aks.tf

@@ -37,6 +37,7 @@ module "prow_build" {
   identity_ids  = [azurerm_user_assigned_identity.aks_identity.id]
 
   msi_auth_for_monitoring_enabled = true
+  tags                            = var.common_tags
 
   kubelet_identity = {
     client_id                 = azurerm_user_assigned_identity.aks_kubelet_identity.client_id
@@ -84,14 +85,15 @@ module "prow_build" {
       vm_size             = "Standard_D8ads_v6"
       enable_auto_scaling = true
       kubelet_disk_type   = "OS"
-      min_count           = 3
+      min_count           = 1
       max_count           = 100
       max_pods            = 110
       os_disk_type        = "Ephemeral"
       os_disk_size_gb     = 100
       os_sku              = "Ubuntu"
-      vnet_subnet_id      = module.prow_network.subnets.prow_build_aks.resource_id
-
+      vnet_subnet = {
+        id = module.prow_network.subnets.prow_build_aks.resource_id
+      }
       upgrade_settings = {
         max_surge                     = "33%"
         drain_timeout_in_minutes      = 90
@@ -103,14 +105,15 @@ module "prow_build" {
       vm_size             = "Standard_D8pds_v6"
       enable_auto_scaling = true
       kubelet_disk_type   = "OS"
-      min_count           = 3
+      min_count           = 1
       max_count           = 100
       max_pods            = 110
       os_disk_type        = "Ephemeral"
       os_disk_size_gb     = 100
       os_sku              = "Ubuntu"
-      vnet_subnet_id      = module.prow_network.subnets.prow_build_aks.resource_id
-
+      vnet_subnet = {
+        id = module.prow_network.subnets.prow_build_aks.resource_id
+      }
       upgrade_settings = {
         max_surge                     = "33%"
         drain_timeout_in_minutes      = 90
@@ -121,13 +124,3 @@ module "prow_build" {
 
   depends_on = [module.prow_network]
 }
-
-# Prevent resource group deletion
-resource "null_resource" "prow_nodepool_rg_tag" {
-
-  provisioner "local-exec" {
-    command = "az group update --resource-group ${module.prow_build.node_resource_group} --tags DO-NOT-DELETE=true"
-  }
-
-  depends_on = [module.prow_build]
-}

infra/azure/terraform/k8s-infra-prow-build/logging.tf

@@ -17,7 +17,8 @@ limitations under the License.
 resource "azurerm_log_analytics_workspace_table" "this" {
   for_each = toset(local.log_analytics_tables)
 
-  name         = each.value
-  workspace_id = module.prow_build.azurerm_log_analytics_workspace_id
-  plan         = "Basic"
+  name                    = each.value
+  workspace_id            = module.prow_build.azurerm_log_analytics_workspace_id
+  plan                    = "Basic"
+  total_retention_in_days = 30
 }

infra/azure/terraform/k8s-infra-prow-build/rbac.tf

@@ -14,12 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-resource "azurerm_role_assignment" "admin" {
-  role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin"
-  scope                = module.prow_build.aks_id
-  principal_id         = data.azurerm_client_config.current.object_id # Me
-}
-
 # Control Plane
 
 resource "azurerm_role_assignment" "control_plane_mi" {
@@ -75,7 +69,7 @@ resource "azurerm_federated_identity_credential" "aks_admin_prow" {
 }
 
 resource "azurerm_role_assignment" "aks_admin" {
-  role_definition_name = "Azure Arc Kubernetes Cluster Admin"
+  role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin"
   scope                = azurerm_resource_group.rg.id
-  principal_id         = azurerm_user_assigned_identity.aks_kubelet_identity.principal_id
+  principal_id         = azurerm_user_assigned_identity.aks_admin.principal_id
 }

infra/gcp/terraform/k8s-infra-prow-build/iam.tf

@@ -34,6 +34,8 @@ module "iam" {
       "principal://iam.googleapis.com/projects/${module.project.project_number}/locations/global/workloadIdentityPools/${module.project.project_id}.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
       "principal://iam.googleapis.com/projects/180382678033/locations/global/workloadIdentityPools/k8s-infra-prow-build-trusted.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
       "principal://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/k8s-infra-prow.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
+      "principal://iam.googleapis.com/${google_iam_workload_identity_pool.eks_cluster.name}/subject/ns/external-secrets/sa/external-secrets",
+      "principal://iam.googleapis.com/${google_iam_workload_identity_pool.aks_cluster.name}/subject/ns/external-secrets/sa/external-secrets",
     ]
   }
 }
@@ -55,7 +57,10 @@ resource "google_iam_workload_identity_pool_provider" "eks_cluster" {
   workload_identity_pool_id          = google_iam_workload_identity_pool.eks_cluster.workload_identity_pool_id
   workload_identity_pool_provider_id = "oidc"
   attribute_mapping = {
-    "google.subject" = "assertion.sub"
+    "google.subject"                 = "\"ns/\" + assertion['kubernetes.io']['namespace'] + \"/sa/\" + assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.namespace"            = "assertion['kubernetes.io']['namespace']"
+    "attribute.service_account_name" = "assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.pod"                  = "assertion['kubernetes.io']['pod']['name']"
   }
   oidc {
     # From EKS cluster created in https://github.com/kubernetes/k8s.io/tree/main/infra/aws/terraform/prow-build-cluster
@@ -71,7 +76,10 @@ resource "google_iam_workload_identity_pool_provider" "eks_kops" {
   workload_identity_pool_id          = google_iam_workload_identity_pool.eks_cluster.workload_identity_pool_id
   workload_identity_pool_provider_id = "kops"
   attribute_mapping = {
-    "google.subject" = "assertion.sub"
+    "google.subject"                 = "\"ns/\" + assertion['kubernetes.io']['namespace'] + \"/sa/\" + assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.namespace"            = "assertion['kubernetes.io']['namespace']"
+    "attribute.service_account_name" = "assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.pod"                  = "assertion['kubernetes.io']['pod']['name']"
   }
   oidc {
     # From EKS cluster created in https://github.com/kubernetes/k8s.io/tree/main/infra/aws/terraform/kops-infra-ci
@@ -97,7 +105,10 @@ resource "google_iam_workload_identity_pool_provider" "aks_cluster" {
   workload_identity_pool_id          = google_iam_workload_identity_pool.aks_cluster.workload_identity_pool_id
   workload_identity_pool_provider_id = "oidc"
   attribute_mapping = {
-    "google.subject" = "assertion.sub"
+    "google.subject"                 = "\"ns/\" + assertion['kubernetes.io']['namespace'] + \"/sa/\" + assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.namespace"            = "assertion['kubernetes.io']['namespace']"
+    "attribute.service_account_name" = "assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.pod"                  = "assertion['kubernetes.io']['pod']['name']"
   }
   oidc {
     # From AKS cluster created in https://github.com/kubernetes/k8s.io/tree/main/infra/azure/terraform/k8s-infra-prow-build

infra/gcp/terraform/k8s-infra-prow-build/peering.tf

@@ -22,4 +22,11 @@ resource "google_vmwareengine_network_peering" "gvce_peering" {
   vmware_engine_network               = "projects/broadcom-451918/locations/global/vmwareEngineNetworks/k8s-gcp-gcve-network"
   export_custom_routes_with_public_ip = true
   import_custom_routes_with_public_ip = true
+  lifecycle {
+    ignore_changes = [
+      # https://github.com/hashicorp/terraform-provider-google/issues/17817
+      export_custom_routes_with_public_ip,
+      import_custom_routes_with_public_ip,
+    ]
+  }
 }

infra/gcp/terraform/k8s-infra-prow-build/serviceaccounts.tf

@@ -30,8 +30,8 @@ locals {
       project_roles     = ["roles/secretmanager.secretAccessor"],
       cluster_namespace = "kubernetes-external-secrets"
       additional_workload_identity_principals = [
-        "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.eks_cluster.name}/*",
-        "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.aks_cluster.name}/*"
+        "principal://iam.googleapis.com/${google_iam_workload_identity_pool.eks_cluster.name}/subject/ns/external-secrets/sa/external-secrets",
+        "principal://iam.googleapis.com/${google_iam_workload_identity_pool.aks_cluster.name}/subject/ns/external-secrets/sa/external-secrets"
       ]
     }
   }

infra/gcp/terraform/k8s-infra-prow/buckets.tf

@@ -120,6 +120,11 @@ module "prow_bucket" {
       role   = "roles/storage.objectAdmin"
       member = "principalSet://iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/ibm-clusters/attribute.namespace/test-pods"
     },
+    {
+      // AKS build clusters, pods in the test-pods namespace only
+      role   = "roles/storage.objectAdmin"
+      member = "principalSet://iam.googleapis.com/projects/773781448124/locations/global/workloadIdentityPools/prow-aks/attribute.namespace/test-pods"
+    },
     {
       role   = "roles/storage.objectViewer"
       member = "allUsers"

kubernetes/aks-prow-build/prow/kyverno.yaml

@@ -22,25 +22,29 @@ spec:
               # pod order matters
               - name: clonerefs
               - (name): "initupload"
-                # prow passes the json path directly, uncomment this once the feature is disabled in prow
-                # env:
-                #   - name: GOOGLE_APPLICATION_CREDENTIALS
-                #     value: /secrets/gcs/service-account.json
+                env:
+                  - name: GOOGLE_APPLICATION_CREDENTIALS
+                    value: /etc/google/adc.json
                 volumeMounts:
                   - mountPath: /var/run/secrets/google-iam-token/serviceaccount
                     name: google-iam-token
                     readOnly: true
+                  - mountPath: /etc/google
+                    name: google-adc
+                    readOnly: true
             containers:
               - name: test
               - (name): sidecar
-                # prow passes the json path directly, uncomment this once the feature is disabled in prow
-                # env:
-                #   - name: GOOGLE_APPLICATION_CREDENTIALS
-                #     value: /secrets/gcs/service-account.json
+                env:
+                  - name: GOOGLE_APPLICATION_CREDENTIALS
+                    value: /etc/google/adc.json
                 volumeMounts:
                   - mountPath: /var/run/secrets/google-iam-token/serviceaccount
                     name: google-iam-token
                     readOnly: true
+                  - mountPath: /etc/google
+                    name: google-adc
+                    readOnly: true
             volumes:
               - name: google-iam-token
                 projected:
@@ -50,3 +54,26 @@ spec:
                         audience: sts.googleapis.com
                         expirationSeconds: 86400
                         path: token
+              - name: google-adc
+                configMap:
+                  name: google-adc
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: google-adc
+data:
+  adc.json: |
+    {
+      "universe_domain": "googleapis.com",
+      "type": "external_account",
+      "audience": "//iam.googleapis.com/projects/773781448124/locations/global/workloadIdentityPools/prow-aks/providers/oidc",
+      "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+      "token_url": "https://sts.googleapis.com/v1/token",
+      "credential_source": {
+        "file": "/var/run/secrets/google-iam-token/serviceaccount/token",
+        "format": {
+          "type": "text"
+        }
+      }
+    }

kubernetes/apps/kyverno.yaml

@@ -9,6 +9,13 @@ spec:
         selector:
           matchLabels:
             cloud: ibm
+    - clusters:
+        selector:
+          matchExpressions:
+            - key: name
+              operator: In
+              values:
+                - aks-prow-build
   template:
     metadata:
       name: "kyverno-{{ .name }}"
@@ -20,7 +27,7 @@ spec:
       sources:
         - chart: kyverno
           repoURL: "https://kyverno.github.io/kyverno"
-          targetRevision: 3.5.1
+          targetRevision: 3.6.1
           helm:
             releaseName: kyverno
             valueFiles:

kubernetes/apps/prow.yaml

@@ -3,6 +3,9 @@ kind: ApplicationSet
 metadata:
   name: prow
 spec:
+  ignoreApplicationDifferences:
+    - jsonPointers:
+        - /spec/syncPolicy
   goTemplate: true
   generators:
     - clusters:
@@ -28,3 +31,5 @@ spec:
         automated:
           prune: true
           selfHeal: true
+        syncOptions:
+          - CreateNamespace=true

kubernetes/gke-utility/argocd/clusters.yaml

@@ -207,6 +207,7 @@ kind: Secret
 metadata:
   name: aks-prow-build
   labels:
+    name: aks-prow-build
     argocd.argoproj.io/secret-type: cluster
     clusterType: prow
     environment: prod
@@ -223,7 +224,7 @@ stringData:
         "env": {
           "AAD_ENVIRONMENT_NAME": "AzurePublicCloud",
           "AZURE_CLIENT_ID": "92ce800b-0477-44c1-ac44-aa53fb973993",
-          "AZURE_TENANT_ID": "d1aa7522-0959-442e-80ee-8c4f7fb4c184"
+          "AZURE_TENANT_ID": "d1aa7522-0959-442e-80ee-8c4f7fb4c184",
           "AZURE_FEDERATED_TOKEN_FILE": "/var/run/secrets/azure-token/serviceaccount/token",
           "AZURE_AUTHORITY_HOST": "https://login.microsoftonline.com/",
           "AAD_LOGIN_METHOD": "workloadidentity"