tf-multienv-cicd-anthos-autopilot Terraform recreates/updates resources every run

[arueth]

Describe the bug

When running the tf-multienv-cicd-anthos-autopilot Terraform, multiple resources are recreated/updated each run due to perceived changes.

To Reproduce

1. run terraform apply
2. run terraform apply again

Additional context

This seems to be a byproduct of the modules used and not having proper lifecycle ignore_changes set.

Exposure

Persistent

[arueth]

Note: Objects have changed outside of Terraform

Terraform detected the following changes made outside of Terraform since the last "terraform apply" which may have affected this plan:

  # module.cloudsql_production.google_sql_database_instance.default has changed
  ~ resource "google_sql_database_instance" "default" {
        id                             = "bank-of-anthos-db-production"
        name                           = "bank-of-anthos-db-production"
        # (14 unchanged attributes hidden)

      ~ settings {
          ~ version                     = 4 -> 5
            # (11 unchanged attributes hidden)

            # (4 unchanged blocks hidden)
        }

        # (1 unchanged block hidden)
    }


Unless you have made equivalent changes to your configuration, or ignored the relevant attributes using ignore_changes, the following plan may include actions to undo or respond to these changes.

──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────

Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
  + create
  ~ update in-place
-/+ destroy and then create replacement

Terraform will perform the following actions:

  # google_gke_hub_membership.development must be replaced
-/+ resource "google_gke_hub_membership" "development" {
      ~ id            = "projects/rueth-development/locations/global/memberships/development-membership" -> (known after apply)
      - labels        = {} -> null
      ~ name          = "projects/rueth-development/locations/global/memberships/development-membership" -> (known after apply)
        # (2 unchanged attributes hidden)

      ~ authority {
          ~ issuer = "https://container.googleapis.com/v1/projects/rueth-development/locations/us-west1/clusters/development" # forces replacement -> (known after apply) # forces replacement
        }

      ~ endpoint {
          ~ gke_cluster {
              ~ resource_link = "//container.googleapis.com/projects/rueth-development/locations/us-west1/clusters/development" # forces replacement -> (known after apply) # forces replacement
            }
        }
    }

  # module.ci-cd-pipeline["accounts/contacts"].google_artifact_registry_repository_iam_member.cloud_build will be created
  + resource "google_artifact_registry_repository_iam_member" "cloud_build" {
      + etag       = (known after apply)
      + id         = (known after apply)
      + location   = "us-west1"
      + member     = "serviceAccount:ci-contacts@rueth-development.iam.gserviceaccount.com"
      + project    = "rueth-development"
      + repository = "bank-of-anthos"
      + role       = "roles/artifactregistry.writer"
    }

  # module.ci-cd-pipeline["accounts/contacts"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/contacts"
        name        = "accounts/contacts"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["accounts/userservice"].google_artifact_registry_repository_iam_member.cloud_build will be created
  + resource "google_artifact_registry_repository_iam_member" "cloud_build" {
      + etag       = (known after apply)
      + id         = (known after apply)
      + location   = "us-west1"
      + member     = "serviceAccount:ci-userservice@rueth-development.iam.gserviceaccount.com"
      + project    = "rueth-development"
      + repository = "bank-of-anthos"
      + role       = "roles/artifactregistry.writer"
    }

  # module.ci-cd-pipeline["accounts/userservice"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/userservice"
        name        = "accounts/userservice"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["frontend"].google_artifact_registry_repository_iam_member.cloud_build will be created
  + resource "google_artifact_registry_repository_iam_member" "cloud_build" {
      + etag       = (known after apply)
      + id         = (known after apply)
      + location   = "us-west1"
      + member     = "serviceAccount:ci-frontend@rueth-development.iam.gserviceaccount.com"
      + project    = "rueth-development"
      + repository = "bank-of-anthos"
      + role       = "roles/artifactregistry.writer"
    }

  # module.ci-cd-pipeline["frontend"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/frontend"
        name        = "frontend"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["ledger/balancereader"].google_artifact_registry_repository_iam_member.cloud_build will be created
  + resource "google_artifact_registry_repository_iam_member" "cloud_build" {
      + etag       = (known after apply)
      + id         = (known after apply)
      + location   = "us-west1"
      + member     = "serviceAccount:ci-balancereader@rueth-development.iam.gserviceaccount.com"
      + project    = "rueth-development"
      + repository = "bank-of-anthos"
      + role       = "roles/artifactregistry.writer"
    }

  # module.ci-cd-pipeline["ledger/balancereader"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/balancereader"
        name        = "ledger/balancereader"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["ledger/ledgerwriter"].google_artifact_registry_repository_iam_member.cloud_build will be created
  + resource "google_artifact_registry_repository_iam_member" "cloud_build" {
      + etag       = (known after apply)
      + id         = (known after apply)
      + location   = "us-west1"
      + member     = "serviceAccount:ci-ledgerwriter@rueth-development.iam.gserviceaccount.com"
      + project    = "rueth-development"
      + repository = "bank-of-anthos"
      + role       = "roles/artifactregistry.writer"
    }

  # module.ci-cd-pipeline["ledger/ledgerwriter"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/ledgerwriter"
        name        = "ledger/ledgerwriter"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["ledger/transactionhistory"].google_artifact_registry_repository_iam_member.cloud_build will be created
  + resource "google_artifact_registry_repository_iam_member" "cloud_build" {
      + etag       = (known after apply)
      + id         = (known after apply)
      + location   = "us-west1"
      + member     = "serviceAccount:ci-transactionhistory@rueth-development.iam.gserviceaccount.com"
      + project    = "rueth-development"
      + repository = "bank-of-anthos"
      + role       = "roles/artifactregistry.writer"
    }

  # module.ci-cd-pipeline["ledger/transactionhistory"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/transactionhistory"
        name        = "ledger/transactionhistory"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.gke_development.google_container_cluster.primary must be replaced
-/+ resource "google_container_cluster" "primary" {
      ~ cluster_ipv4_cidr           = "172.16.0.0/16" -> (known after apply)
      ~ datapath_provider           = "ADVANCED_DATAPATH" -> (known after apply)
      ~ default_max_pods_per_node   = 110 -> (known after apply)
      ~ enable_tpu                  = false -> (known after apply)
      ~ endpoint                    = "10.6.0.2" -> (known after apply)
      ~ id                          = "projects/rueth-development/locations/us-west1/clusters/development" -> (known after apply)
      - initial_node_count          = 0 -> null
      ~ label_fingerprint           = "dc0aa57c" -> (known after apply)
      ~ logging_service             = "logging.googleapis.com/kubernetes" -> (known after apply)
      ~ master_version              = "1.27.3-gke.100" -> (known after apply)
      ~ monitoring_service          = "monitoring.googleapis.com/kubernetes" -> (known after apply)
        name                        = "development"
      ~ node_version                = "1.27.3-gke.100" -> (known after apply)
      + operation                   = (known after apply)
      + private_ipv6_google_access  = (known after apply)
      ~ self_link                   = "https://container.googleapis.com/v1beta1/projects/rueth-development/locations/us-west1/clusters/development" -> (known after apply)
      ~ services_ipv4_cidr          = "172.17.0.0/16" -> (known after apply)
      + tpu_ipv4_cidr_block         = (known after apply)
        # (15 unchanged attributes hidden)

      ~ addons_config {
          - dns_cache_config {
              - enabled = true -> null
            }
          - gce_persistent_disk_csi_driver_config {
              - enabled = true -> null
            }
          - gcp_filestore_csi_driver_config {
              - enabled = true -> null
            }
          - gcs_fuse_csi_driver_config {
              - enabled = true -> null
            }
          - network_policy_config {
              - disabled = true -> null
            }

            # (2 unchanged blocks hidden)
        }

      - binary_authorization {
          - enabled = false -> null
        }

      ~ cluster_autoscaling {
          - autoscaling_profile = "OPTIMIZE_UTILIZATION" -> null
          ~ enabled             = true -> (known after apply)

          - auto_provisioning_defaults {
              - disk_size       = 0 -> null
              - image_type      = "COS_CONTAINERD" -> null
              - oauth_scopes    = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - service_account = "default" -> null

              - management {
                  - auto_repair     = true -> null
                  - auto_upgrade    = true -> null
                  - upgrade_options = [] -> null
                }

              - upgrade_settings {
                  - max_surge       = 1 -> null
                  - max_unavailable = 0 -> null
                  - strategy        = "SURGE" -> null
                }
            }

          - resource_limits {
              - maximum       = 1000000000 -> null
              - minimum       = 0 -> null
              - resource_type = "cpu" -> null
            }
          - resource_limits {
              - maximum       = 1000000000 -> null
              - minimum       = 0 -> null
              - resource_type = "memory" -> null
            }
          - resource_limits {
              - maximum       = 1000000000 -> null
              - minimum       = 0 -> null
              - resource_type = "nvidia-tesla-t4" -> null
            }
          - resource_limits {
              - maximum       = 1000000000 -> null
              - minimum       = 0 -> null
              - resource_type = "nvidia-tesla-a100" -> null
            }
        }

      - cluster_telemetry {
          - type = "ENABLED" -> null
        }

      - dns_config { # forces replacement
          - cluster_dns        = "CLOUD_DNS" -> null
          - cluster_dns_domain = "cluster.local" -> null
          - cluster_dns_scope  = "CLUSTER_SCOPE" -> null
        }

      - gateway_api_config {
          - channel = "CHANNEL_STANDARD" -> null
        }

      ~ ip_allocation_policy {
          ~ cluster_ipv4_cidr_block       = "172.16.0.0/16" -> (known after apply)
          ~ services_ipv4_cidr_block      = "172.17.0.0/16" -> (known after apply)
            # (3 unchanged attributes hidden)

          - pod_cidr_overprovision_config {
              - disabled = false -> null
            }
        }

      - logging_config {
          - enable_components = [
              - "SYSTEM_COMPONENTS",
              - "WORKLOADS",
            ] -> null
        }

      ~ maintenance_policy {
          ~ daily_maintenance_window {
              ~ duration   = "PT4H0M0S" -> (known after apply)
                # (1 unchanged attribute hidden)
            }
        }

      ~ master_auth {
          + client_certificate     = (known after apply)
          + client_key             = (sensitive value)
          ~ cluster_ca_certificate = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMVENDQXBXZ0F3SUJBZ0lSQU55dG1ybHRJUnZxbnlrMTZTcmZmemd3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa05EVTNNV1JpTkRFdFlXUmxNaTAwWkdKakxUaGtNR0V0WmpJNVl6VTRPV1ZoTVRneQpNQ0FYRFRJek1EZ3dOekl5TVRVMU9Wb1lEekl3TlRNd056TXdNak14TlRVNVdqQXZNUzB3S3dZRFZRUURFeVEwCk5UY3haR0kwTVMxaFpHVXlMVFJrWW1NdE9HUXdZUzFtTWpsak5UZzVaV0V4T0RJd2dnR2lNQTBHQ1NxR1NJYjMKRFFFQkFRVUFBNElCandBd2dnR0tBb0lCZ1FDcmhVUGtML2xLVWVTczhTcHJwVVI0b2s0a0o1MlBKQW1VL2lBdgpmcHkwWTdsRjFweUFlR1RWQkJqcG4wQzNJK1JrTXNDYjlWelZBWGRTWGdBRzBqclJOOXFLeDdnYitLNU40Y2pRCnUvZ3BZUXhWOXhWeTllQXIraHd0SGNKaXBwWkdxY29iSkZ1Qmo5elg1b3hmbEpPRTZXdFRJdExEdC9ueW56bzkKbGNKdmJvQnFlZFVJYUxvclFIRzZ5aEN6V2tuM2lyYzJkb0RnUE8rTTltQmxHeVZQRHVMM1B6L05kQ28rUW9UNApWeEFBb29tR081VlFxK2FoUDJzN0Z5SUNVblFKSmFsSHd3eFNHYXhJUUlDQXUyemFYdnFLb0lFaG1BaEYxbVp5CktpVDNCTVYzZC9nSzhnMTJCemxSYlpTMzZmY1ptTEhVaUVyby9OeWUwWFBUREYwQmFzYlYrTkV3Q3F4eGFxckEKZ1BKMmJ1MGhGWHAwYTY5ekFiNWRZNVRRZXRpbzFTZTUxVnFDaFB6ZDlnb0VZY1poNllybEdzQ1Vnc25rcGpoeApUYmxodUJpc2V4T1RiVDRxcUJkSEc4MXh0YklMWFdPUWZZeEF0YXg0OFJZVG5KT3M3RlpZMzhmaHROTjUxMFA3CmdWUms0NUZZK1drZ1poaURNSEltMjBzOUl4OENBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdJRU1BOEcKQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGS1N0WTQrSmhLbWpDcmtLM2U2bkY4VnI3Y2dXTUEwRwpDU3FHU0liM0RRRUJDd1VBQTRJQmdRQllBaGFvMElMbnNEY2dUSW5IalJnSkFFNHByeGtZUXozM01tZjVFMTlkCnlUT0UzQTFYYmtUVE1iZnBOMVAvZ0QyMGNxaUtZcEEwZ0pRQkdFbWd0L3dqWE0xTG1wdjlEbnVjZDhuYmFWdkEKdHllamxJTVd5MWRxbUkxUGlyNUwzeEdvSGVodUphdklGak1oYThsZ2RVUHpuc3ZUVzYvOUF5aitwN0ZKb2FHVwo1YnFuaW1lQjBsMHRHS2I2VHk4Q1hjMWYzNjJ3Q0pMdU9JdEk1NEtKS3MwbUd0RFE5T1c1eEd5a0Z6dWcvaHQyCjBPUzc5dUNYdVU2T0daL29ySXB6MFZOMTFmdXpKUnNDVlZoTlh0Q0o2MVcvUXFXWE01RjY5bGRzZzVoa2JyT2cKL2pxK1U1aXF5aUczOWYyd1B1cmlRaW5RQVJHSDVMbnBmcXdCNW9yZUlwbEZYd1BGYjVBaW1jOWoxY3hLZHdlaApxWnVQamZOQm5qWkR0a0M1OUdwVmdWcCs1SDVKUkFkN1NLeDdXank4RlM2VkF2c29PSXBXTU5CaFlobzIxVWRJCjJFZldQYWtTSW1CbGRObmFPTEp6a0lqdEp2dVVVU2FpNmh4MVBCb1dRMWxYUjIzUXVkbnE5UUdJcmlXMWkrS2UKaXhBbnc2QndodWo3ZE81Z0U2QmVTZ1E9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" -> (known after apply)

            # (1 unchanged block hidden)
        }

      ~ master_authorized_networks_config {
          ~ gcp_public_cidrs_access_enabled = false -> (known after apply)

            # (1 unchanged block hidden)
        }

      - monitoring_config {
          - enable_components = [
              - "SYSTEM_COMPONENTS",
            ] -> null

          - managed_prometheus {
              - enabled = true -> null
            }
        }

      - network_policy {
          - enabled  = false -> null
          - provider = "PROVIDER_UNSPECIFIED" -> null
        }

      - node_config {
          - disk_size_gb      = 100 -> null
          - disk_type         = "pd-standard" -> null
          - guest_accelerator = [] -> null
          - image_type        = "COS_CONTAINERD" -> null
          - labels            = {} -> null
          - local_ssd_count   = 0 -> null
          - logging_variant   = "DEFAULT" -> null
          - machine_type      = "e2-medium" -> null
          - metadata          = {
              - "disable-legacy-endpoints" = "true"
            } -> null
          - oauth_scopes      = [
              - "https://www.googleapis.com/auth/devstorage.read_only",
              - "https://www.googleapis.com/auth/logging.write",
              - "https://www.googleapis.com/auth/monitoring",
              - "https://www.googleapis.com/auth/service.management.readonly",
              - "https://www.googleapis.com/auth/servicecontrol",
              - "https://www.googleapis.com/auth/trace.append",
            ] -> null
          - preemptible       = false -> null
          - resource_labels   = {} -> null
          - service_account   = "default" -> null
          - spot              = false -> null
          - tags              = [] -> null
          - taint             = [] -> null

          - reservation_affinity {
              - consume_reservation_type = "NO_RESERVATION" -> null
              - values                   = [] -> null
            }

          - shielded_instance_config {
              - enable_integrity_monitoring = true -> null
              - enable_secure_boot          = true -> null
            }

          - workload_metadata_config {
              - mode = "GKE_METADATA" -> null
            }
        }

      - node_pool {
          - initial_node_count          = 1 -> null
          - instance_group_urls         = [
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-c/instanceGroupManagers/gk3-development-default-pool-f448374d-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-b/instanceGroupManagers/gk3-development-default-pool-de996095-grp",
            ] -> null
          - managed_instance_group_urls = [
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-c/instanceGroups/gk3-development-default-pool-f448374d-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-b/instanceGroups/gk3-development-default-pool-de996095-grp",
            ] -> null
          - max_pods_per_node           = 32 -> null
          - name                        = "default-pool" -> null
          - node_count                  = 1 -> null
          - node_locations              = [
              - "us-west1-b",
              - "us-west1-c",
            ] -> null
          - version                     = "1.27.3-gke.100" -> null

          - autoscaling {
              - location_policy      = "BALANCED" -> null
              - max_node_count       = 1000 -> null
              - min_node_count       = 0 -> null
              - total_max_node_count = 0 -> null
              - total_min_node_count = 0 -> null
            }

          - management {
              - auto_repair  = true -> null
              - auto_upgrade = true -> null
            }

          - network_config {
              - create_pod_range     = false -> null
              - enable_private_nodes = false -> null
              - pod_ipv4_cidr_block  = "172.16.0.0/16" -> null
              - pod_range            = "development-ip-range-pods" -> null
            }

          - node_config {
              - disk_size_gb      = 100 -> null
              - disk_type         = "pd-standard" -> null
              - guest_accelerator = [] -> null
              - image_type        = "COS_CONTAINERD" -> null
              - labels            = {} -> null
              - local_ssd_count   = 0 -> null
              - logging_variant   = "DEFAULT" -> null
              - machine_type      = "e2-medium" -> null
              - metadata          = {
                  - "disable-legacy-endpoints" = "true"
                } -> null
              - oauth_scopes      = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - preemptible       = false -> null
              - resource_labels   = {} -> null
              - service_account   = "default" -> null
              - spot              = false -> null
              - tags              = [] -> null
              - taint             = [] -> null

              - reservation_affinity {
                  - consume_reservation_type = "NO_RESERVATION" -> null
                  - values                   = [] -> null
                }

              - shielded_instance_config {
                  - enable_integrity_monitoring = true -> null
                  - enable_secure_boot          = true -> null
                }

              - workload_metadata_config {
                  - mode = "GKE_METADATA" -> null
                }
            }

          - upgrade_settings {
              - max_surge       = 1 -> null
              - max_unavailable = 0 -> null
              - strategy        = "SURGE" -> null
            }
        }
      - node_pool {
          - initial_node_count          = 0 -> null
          - instance_group_urls         = [
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-a/instanceGroupManagers/gk3-development-pool-1-8576f967-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-b/instanceGroupManagers/gk3-development-pool-1-9bf417c0-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-c/instanceGroupManagers/gk3-development-pool-1-8463e566-grp",
            ] -> null
          - managed_instance_group_urls = [
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-a/instanceGroups/gk3-development-pool-1-8576f967-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-b/instanceGroups/gk3-development-pool-1-9bf417c0-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-c/instanceGroups/gk3-development-pool-1-8463e566-grp",
            ] -> null
          - max_pods_per_node           = 32 -> null
          - name                        = "pool-1" -> null
          - node_count                  = 0 -> null
          - node_locations              = [
              - "us-west1-a",
              - "us-west1-b",
              - "us-west1-c",
            ] -> null
          - version                     = "1.27.3-gke.100" -> null

          - autoscaling {
              - location_policy      = "BALANCED" -> null
              - max_node_count       = 1000 -> null
              - min_node_count       = 0 -> null
              - total_max_node_count = 0 -> null
              - total_min_node_count = 0 -> null
            }

          - management {
              - auto_repair  = true -> null
              - auto_upgrade = true -> null
            }

          - network_config {
              - create_pod_range     = false -> null
              - enable_private_nodes = false -> null
              - pod_ipv4_cidr_block  = "172.16.0.0/16" -> null
              - pod_range            = "development-ip-range-pods" -> null
            }

          - node_config {
              - disk_size_gb      = 100 -> null
              - disk_type         = "pd-standard" -> null
              - guest_accelerator = [] -> null
              - image_type        = "COS_CONTAINERD" -> null
              - labels            = {} -> null
              - local_ssd_count   = 0 -> null
              - logging_variant   = "DEFAULT" -> null
              - machine_type      = "e2-standard-2" -> null
              - metadata          = {
                  - "disable-legacy-endpoints" = "true"
                } -> null
              - oauth_scopes      = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - preemptible       = false -> null
              - resource_labels   = {} -> null
              - service_account   = "default" -> null
              - spot              = false -> null
              - tags              = [] -> null
              - taint             = [] -> null

              - reservation_affinity {
                  - consume_reservation_type = "NO_RESERVATION" -> null
                  - values                   = [] -> null
                }

              - shielded_instance_config {
                  - enable_integrity_monitoring = true -> null
                  - enable_secure_boot          = true -> null
                }

              - workload_metadata_config {
                  - mode = "GKE_METADATA" -> null
                }
            }

          - upgrade_settings {
              - max_surge       = 1 -> null
              - max_unavailable = 0 -> null
              - strategy        = "SURGE" -> null
            }
        }
      - node_pool {
          - initial_node_count          = 0 -> null
          - instance_group_urls         = [
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-a/instanceGroupManagers/gk3-development-pool-2-348d3eb1-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-b/instanceGroupManagers/gk3-development-pool-2-417e9082-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-c/instanceGroupManagers/gk3-development-pool-2-0d0e976e-grp",
            ] -> null
          - managed_instance_group_urls = [
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-a/instanceGroups/gk3-development-pool-2-348d3eb1-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-b/instanceGroups/gk3-development-pool-2-417e9082-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-c/instanceGroups/gk3-development-pool-2-0d0e976e-grp",
            ] -> null
          - max_pods_per_node           = 32 -> null
          - name                        = "pool-2" -> null
          - node_count                  = 0 -> null
          - node_locations              = [
              - "us-west1-a",
              - "us-west1-b",
              - "us-west1-c",
            ] -> null
          - version                     = "1.27.3-gke.100" -> null

          - autoscaling {
              - location_policy      = "BALANCED" -> null
              - max_node_count       = 1000 -> null
              - min_node_count       = 0 -> null
              - total_max_node_count = 0 -> null
              - total_min_node_count = 0 -> null
            }

          - management {
              - auto_repair  = true -> null
              - auto_upgrade = true -> null
            }

          - network_config {
              - create_pod_range     = false -> null
              - enable_private_nodes = false -> null
              - pod_ipv4_cidr_block  = "172.16.0.0/16" -> null
              - pod_range            = "development-ip-range-pods" -> null
            }

          - node_config {
              - disk_size_gb      = 100 -> null
              - disk_type         = "pd-standard" -> null
              - guest_accelerator = [] -> null
              - image_type        = "COS_CONTAINERD" -> null
              - labels            = {} -> null
              - local_ssd_count   = 0 -> null
              - logging_variant   = "DEFAULT" -> null
              - machine_type      = "e2-standard-4" -> null
              - metadata          = {
                  - "disable-legacy-endpoints" = "true"
                } -> null
              - oauth_scopes      = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - preemptible       = false -> null
              - resource_labels   = {} -> null
              - service_account   = "default" -> null
              - spot              = false -> null
              - tags              = [] -> null
              - taint             = [] -> null

              - reservation_affinity {
                  - consume_reservation_type = "NO_RESERVATION" -> null
                  - values                   = [] -> null
                }

              - shielded_instance_config {
                  - enable_integrity_monitoring = true -> null
                  - enable_secure_boot          = true -> null
                }

              - workload_metadata_config {
                  - mode = "GKE_METADATA" -> null
                }
            }

          - upgrade_settings {
              - max_surge       = 1 -> null
              - max_unavailable = 0 -> null
              - strategy        = "SURGE" -> null
            }
        }
      - node_pool {
          - initial_node_count          = 0 -> null
          - instance_group_urls         = [
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-a/instanceGroupManagers/gk3-development-pool-3-c74397b6-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-b/instanceGroupManagers/gk3-development-pool-3-7fc03df5-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-c/instanceGroupManagers/gk3-development-pool-3-b9801cb8-grp",
            ] -> null
          - managed_instance_group_urls = [
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-a/instanceGroups/gk3-development-pool-3-c74397b6-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-b/instanceGroups/gk3-development-pool-3-7fc03df5-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-c/instanceGroups/gk3-development-pool-3-b9801cb8-grp",
            ] -> null
          - max_pods_per_node           = 32 -> null
          - name                        = "pool-3" -> null
          - node_count                  = 0 -> null
          - node_locations              = [
              - "us-west1-a",
              - "us-west1-b",
              - "us-west1-c",
            ] -> null
          - version                     = "1.27.3-gke.100" -> null

          - autoscaling {
              - location_policy      = "BALANCED" -> null
              - max_node_count       = 1000 -> null
              - min_node_count       = 0 -> null
              - total_max_node_count = 0 -> null
              - total_min_node_count = 0 -> null
            }

          - management {
              - auto_repair  = true -> null
              - auto_upgrade = true -> null
            }

          - network_config {
              - create_pod_range     = false -> null
              - enable_private_nodes = false -> null
              - pod_ipv4_cidr_block  = "172.16.0.0/16" -> null
              - pod_range            = "development-ip-range-pods" -> null
            }

          - node_config {
              - disk_size_gb      = 100 -> null
              - disk_type         = "pd-standard" -> null
              - guest_accelerator = [] -> null
              - image_type        = "COS_CONTAINERD" -> null
              - labels            = {} -> null
              - local_ssd_count   = 0 -> null
              - logging_variant   = "DEFAULT" -> null
              - machine_type      = "e2-standard-8" -> null
              - metadata          = {
                  - "disable-legacy-endpoints" = "true"
                } -> null
              - oauth_scopes      = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - preemptible       = false -> null
              - resource_labels   = {} -> null
              - service_account   = "default" -> null
              - spot              = false -> null
              - tags              = [] -> null
              - taint             = [] -> null

              - reservation_affinity {
                  - consume_reservation_type = "NO_RESERVATION" -> null
                  - values                   = [] -> null
                }

              - shielded_instance_config {
                  - enable_integrity_monitoring = true -> null
                  - enable_secure_boot          = true -> null
                }

              - workload_metadata_config {
                  - mode = "GKE_METADATA" -> null
                }
            }

          - upgrade_settings {
              - max_surge       = 1 -> null
              - max_unavailable = 0 -> null
              - strategy        = "SURGE" -> null
            }
        }
      - node_pool {
          - initial_node_count          = 0 -> null
          - instance_group_urls         = [
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-a/instanceGroupManagers/gk3-development-pool-4-9a1d896f-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-b/instanceGroupManagers/gk3-development-pool-4-bb173022-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-c/instanceGroupManagers/gk3-development-pool-4-91154a01-grp",
            ] -> null
          - managed_instance_group_urls = [
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-a/instanceGroups/gk3-development-pool-4-9a1d896f-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-b/instanceGroups/gk3-development-pool-4-bb173022-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-c/instanceGroups/gk3-development-pool-4-91154a01-grp",
            ] -> null
          - max_pods_per_node           = 32 -> null
          - name                        = "pool-4" -> null
          - node_count                  = 0 -> null
          - node_locations              = [
              - "us-west1-a",
              - "us-west1-b",
              - "us-west1-c",
            ] -> null
          - version                     = "1.27.3-gke.100" -> null

          - autoscaling {
              - location_policy      = "BALANCED" -> null
              - max_node_count       = 1000 -> null
              - min_node_count       = 0 -> null
              - total_max_node_count = 0 -> null
              - total_min_node_count = 0 -> null
            }

          - management {
              - auto_repair  = true -> null
              - auto_upgrade = true -> null
            }

          - network_config {
              - create_pod_range     = false -> null
              - enable_private_nodes = false -> null
              - pod_ipv4_cidr_block  = "172.16.0.0/16" -> null
              - pod_range            = "development-ip-range-pods" -> null
            }

          - node_config {
              - disk_size_gb      = 100 -> null
              - disk_type         = "pd-standard" -> null
              - guest_accelerator = [] -> null
              - image_type        = "COS_CONTAINERD" -> null
              - labels            = {} -> null
              - local_ssd_count   = 0 -> null
              - logging_variant   = "DEFAULT" -> null
              - machine_type      = "e2-standard-16" -> null
              - metadata          = {
                  - "disable-legacy-endpoints" = "true"
                } -> null
              - oauth_scopes      = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - preemptible       = false -> null
              - resource_labels   = {} -> null
              - service_account   = "default" -> null
              - spot              = false -> null
              - tags              = [] -> null
              - taint             = [] -> null

              - reservation_affinity {
                  - consume_reservation_type = "NO_RESERVATION" -> null
                  - values                   = [] -> null
                }

              - shielded_instance_config {
                  - enable_integrity_monitoring = true -> null
                  - enable_secure_boot          = true -> null
                }

              - workload_metadata_config {
                  - mode = "GKE_METADATA" -> null
                }
            }

          - upgrade_settings {
              - max_surge       = 1 -> null
              - max_unavailable = 0 -> null
              - strategy        = "SURGE" -> null
            }
        }
      - node_pool {
          - initial_node_count          = 0 -> null
          - instance_group_urls         = [
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-a/instanceGroupManagers/gk3-development-pool-5-16e9d88c-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-b/instanceGroupManagers/gk3-development-pool-5-3356eb65-grp",
              - "https://www.googleapis.com/compute/v1/projects/rueth-development/zones/us-west1-c/instanceGroupManagers/gk3-development-pool-5-ce55a731-grp",
            ] -> null
          - managed_instance_group_urls = [
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-a/instanceGroups/gk3-development-pool-5-16e9d88c-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-b/instanceGroups/gk3-development-pool-5-3356eb65-grp",
              - "https://www.googleapis.com/compute/beta/projects/rueth-development/zones/us-west1-c/instanceGroups/gk3-development-pool-5-ce55a731-grp",
            ] -> null
          - max_pods_per_node           = 32 -> null
          - name                        = "pool-5" -> null
          - node_count                  = 0 -> null
          - node_locations              = [
              - "us-west1-a",
              - "us-west1-b",
              - "us-west1-c",
            ] -> null
          - version                     = "1.27.3-gke.100" -> null

          - autoscaling {
              - location_policy      = "BALANCED" -> null
              - max_node_count       = 1000 -> null
              - min_node_count       = 0 -> null
              - total_max_node_count = 0 -> null
              - total_min_node_count = 0 -> null
            }

          - management {
              - auto_repair  = true -> null
              - auto_upgrade = true -> null
            }

          - network_config {
              - create_pod_range     = false -> null
              - enable_private_nodes = false -> null
              - pod_ipv4_cidr_block  = "172.16.0.0/16" -> null
              - pod_range            = "development-ip-range-pods" -> null
            }

          - node_config {
              - disk_size_gb      = 100 -> null
              - disk_type         = "pd-standard" -> null
              - guest_accelerator = [] -> null
              - image_type        = "COS_CONTAINERD" -> null
              - labels            = {} -> null
              - local_ssd_count   = 0 -> null
              - logging_variant   = "DEFAULT" -> null
              - machine_type      = "e2-standard-32" -> null
              - metadata          = {
                  - "disable-legacy-endpoints" = "true"
                } -> null
              - oauth_scopes      = [
                  - "https://www.googleapis.com/auth/devstorage.read_only",
                  - "https://www.googleapis.com/auth/logging.write",
                  - "https://www.googleapis.com/auth/monitoring",
                  - "https://www.googleapis.com/auth/service.management.readonly",
                  - "https://www.googleapis.com/auth/servicecontrol",
                  - "https://www.googleapis.com/auth/trace.append",
                ] -> null
              - preemptible       = false -> null
              - resource_labels   = {} -> null
              - service_account   = "default" -> null
              - spot              = false -> null
              - tags              = [] -> null
              - taint             = [] -> null

              - reservation_affinity {
                  - consume_reservation_type = "NO_RESERVATION" -> null
                  - values                   = [] -> null
                }

              - shielded_instance_config {
                  - enable_integrity_monitoring = true -> null
                  - enable_secure_boot          = true -> null
                }

              - workload_metadata_config {
                  - mode = "GKE_METADATA" -> null
                }
            }

          - upgrade_settings {
              - max_surge       = 1 -> null
              - max_unavailable = 0 -> null
              - strategy        = "SURGE" -> null
            }
        }

      - node_pool_auto_config {
        }

      - node_pool_defaults {
          - node_config_defaults {
              - logging_variant = "DEFAULT" -> null

              - gcfs_config {
                  - enabled = true -> null
                }
            }
        }

      - pod_security_policy_config {
          - enabled = false -> null
        }

      ~ private_cluster_config {
          ~ peering_name            = "gke-n883aaf3d4ea52e80475-7610-a6da-peer" -> (known after apply)
          ~ private_endpoint        = "10.6.0.2" -> (known after apply)
          ~ public_endpoint         = "34.105.89.79" -> (known after apply)
            # (3 unchanged attributes hidden)

            # (1 unchanged block hidden)
        }

      ~ protect_config {
          ~ workload_vulnerability_mode = "WORKLOAD_VULNERABILITY_MODE_UNSPECIFIED" -> (known after apply)

            # (1 unchanged block hidden)
        }

      - security_posture_config {
          - mode               = "DISABLED" -> null
          - vulnerability_mode = "VULNERABILITY_MODE_UNSPECIFIED" -> null
        }

      - service_external_ips_config {
          - enabled = false -> null
        }

      - workload_identity_config {
          - workload_pool = "rueth-development.svc.id.goog" -> null
        }

        # (6 unchanged blocks hidden)
    }

Plan: 8 to add, 6 to change, 2 to destroy.

[arueth]

Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
  ~ update in-place

Terraform will perform the following actions:

  # module.artifact-registry-repository-iam-bindings.google_artifact_registry_repository_iam_binding.artifact_registry_iam_authoritative["default--roles/artifactregistry.writer"] will be updated in-place
  ~ resource "google_artifact_registry_repository_iam_binding" "artifact_registry_iam_authoritative" {
        id         = "projects/rueth-development/locations/us-west1/repositories/bank-of-anthos/roles/artifactregistry.writer"
      ~ members    = [
          - "serviceAccount:ci-balancereader@rueth-development.iam.gserviceaccount.com",
          - "serviceAccount:ci-contacts@rueth-development.iam.gserviceaccount.com",
          - "serviceAccount:ci-frontend@rueth-development.iam.gserviceaccount.com",
          - "serviceAccount:ci-ledgerwriter@rueth-development.iam.gserviceaccount.com",
          - "serviceAccount:ci-transactionhistory@rueth-development.iam.gserviceaccount.com",
          - "serviceAccount:ci-userservice@rueth-development.iam.gserviceaccount.com",
            # (1 unchanged element hidden)
        ]
        # (5 unchanged attributes hidden)
    }

  # module.ci-cd-pipeline["accounts/contacts"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/contacts"
        name        = "accounts/contacts"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["accounts/userservice"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/userservice"
        name        = "accounts/userservice"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["frontend"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/frontend"
        name        = "frontend"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["ledger/balancereader"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/balancereader"
        name        = "ledger/balancereader"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["ledger/ledgerwriter"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/ledgerwriter"
        name        = "ledger/ledgerwriter"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

  # module.ci-cd-pipeline["ledger/transactionhistory"].google_clouddeploy_delivery_pipeline.delivery-pipeline will be updated in-place
  ~ resource "google_clouddeploy_delivery_pipeline" "delivery-pipeline" {
        id          = "projects/rueth-development/locations/us-west1/deliveryPipelines/transactionhistory"
        name        = "ledger/transactionhistory"
        # (10 unchanged attributes hidden)

      ~ serial_pipeline {
          ~ stages {
                # (2 unchanged attributes hidden)

              ~ strategy {
                  + standard {
                      + verify = false
                    }
                }
            }

            # (1 unchanged block hidden)
        }
    }

Plan: 0 to add, 7 to change, 0 to destroy.