Skip to content

Commit ea97472

Browse files
authored
Merge pull request #9053 from upodroid/switch-krel-to-community-project
migrate krel to community owned gcp project
2 parents de83bec + 00f063e commit ea97472

18 files changed

Lines changed: 429 additions & 289 deletions

File tree

infra/gcp/bash/ensure-release-projects.sh

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -135,68 +135,3 @@ for PROJECT; do
135135

136136
color 6 "Done"
137137
done
138-
139-
## Special case: setup buckets that are used by CI
140-
141-
# Ensure the given GCS bucket exists in the given project with auto-deletion
142-
# enabled after a default or optionally specified number of days, and
143-
# appropriate permissions for prow, on-call, and release-managers
144-
#
145-
# $1: The GCP project (e.g. k8s-release)
146-
# $2: The GCS bucket (e.g. gs://k8s-release-dev)
147-
# [$3]: The number of days after which objects are auto-delete (e.g. 14, default: 90)
148-
function ensure_kubernetes_ci_gcs_bucket() {
149-
if [ $# -lt 2 ] || [ $# -gt 4 ] || [ -z "$1" ] || [ -z "$2" ] || [ -z "${3:-"x"}" ]; then
150-
echo "${FUNCNAME[0]}(project, gcs_bucket, [auto_deletion_days])" >&2
151-
return 1
152-
fi
153-
local project="${1}"
154-
local bucket="${2}"
155-
local auto_deletion_days="${3:-"90"}"
156-
157-
color 6 "Ensuring ${bucket} exists and is world readable in project: ${project}"
158-
ensure_public_gcs_bucket "${project}" "${bucket}"
159-
160-
color 6 "Ensuring ${bucket} has auto-deletion of ${auto_deletion_days} days"
161-
ensure_gcs_bucket_auto_deletion "${bucket}" "${auto_deletion_days}"
162-
163-
color 6 "Ensuring GCS admins can admin ${bucket} in project: ${project}"
164-
empower_gcs_admins "${project}" "${bucket}"
165-
166-
color 6 "Ensuring prow on-call can admin ${bucket} in project: ${project}"
167-
empower_group_to_admin_gcs_bucket "k8s-infra-prow-oncall@kubernetes.io" "${bucket}"
168-
169-
color 6 "Ensuring prow service account ${PROW_BUILD_SERVICE_ACCOUNT} can write to ${bucket} in project: ${project}"
170-
empower_svcacct_to_write_gcs_bucket "${PROW_BUILD_SERVICE_ACCOUNT}" "${bucket}"
171-
172-
# Empower prow jobs running on google.com-owned k8s-prow or k8s-prow-builds
173-
# clusters to write CI artifacts to the bucket
174-
# TODO(spiffxp): remove this once we've migrated the jobs that rely on this account
175-
# to community-owned build cluster(s)
176-
color 6 "Ensuring prow service account ${PR_KUBEKINS_SERVICE_ACCOUNT} can write to ${bucket} in project: ${project}"
177-
empower_svcacct_to_write_gcs_bucket "${PR_KUBEKINS_SERVICE_ACCOUNT}" "${bucket}"
178-
179-
# Enable access logs to identify what pr-kubekins writes to this bucket
180-
# TODO(spiffxp): consider disabling this once migration is complete
181-
color 6 "Ensuring GCS access logs enabled for ${bucket} in project: ${project}"
182-
ensure_gcs_bucket_logging "${bucket}"
183-
184-
# TODO(spiffxp): I'm not actually sure this makes sense. These groups don't
185-
# have permissions to do this with the google.com-owned bucket
186-
# today. These buckets should be strictly-CI unless there are
187-
# very exceptional circumstances (which is when I'd suggest we
188-
# escalate to the admins above)
189-
for group in ${RELEASE_ADMINS} ${RELEASE_MANAGERS}; do
190-
color 6 "Ensuring group ${group} can write to ${bucket} in project: ${project}"
191-
empower_group_to_write_gcs_bucket "${group}" "${bucket}"
192-
done
193-
194-
}
195-
196-
function special_case_kubernetes_ci_buckets() {
197-
# community-owned equivalent to gs://kubernetes-release-dev
198-
ensure_kubernetes_ci_gcs_bucket "k8s-release" "gs://k8s-release-dev"
199-
}
200-
201-
color 3 "Special case: ensuring GCS buckets for kubernetes CI artifacts exist"
202-
special_case_kubernetes_ci_buckets 2>&1 | indent

infra/gcp/bash/ensure-staging-storage.sh

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ readonly RELEASE_STAGING_PROJECTS=(
4949
"$(k8s_infra_project staging k8s-staging-ci-images)"
5050
"$(k8s_infra_project staging k8s-staging-cip-test)"
5151
"$(k8s_infra_project staging k8s-staging-experimental)"
52-
"$(k8s_infra_project staging k8s-staging-kubernetes)"
5352
"$(k8s_infra_project staging k8s-staging-releng)"
5453
"$(k8s_infra_project staging k8s-staging-releng-test)"
5554
"$(k8s_infra_project staging k8s-staging-publishing-bot)"
@@ -360,24 +359,6 @@ function ensure_release_manager_special_cases() {
360359
color 6 "Empowering ${RELEASE_VIEWERS} as project viewers in ${project}"
361360
ensure_project_role_binding "${project}" "group:${RELEASE_VIEWERS}" "roles/viewer"
362361

363-
# For k8s-staging-kubernetes, grant the kubernetes-release-test (old
364-
# staging) GCB service account admin GCR access to the new staging
365-
# project for Kubernetes releases. This is required for VDF as we need
366-
# to continue running stages/releases from the old project while
367-
# publishing container images to new project.
368-
# ref: https://github.com/kubernetes/release/pull/1230
369-
if [[ "${project}" == "k8s-staging-kubernetes" ]]; then
370-
color 6 "Empowering kubernetes-release-test GCB service account to admin GCR"
371-
empower_svcacct_to_admin_gcr "648026197307@cloudbuild.gserviceaccount.com" "${project}"
372-
fi
373-
374-
# Artifact Registry
375-
#
376-
# Enable Google Artifact Registry to allow Release Managers to prepare
377-
# for GCR to Artifact Registry migration
378-
# ref: https://github.com/kubernetes/k8s.io/issues/1343
379-
ensure_services "${project}" artifactregistry.googleapis.com
380-
381362
# Roles: https://cloud.google.com/artifact-registry/docs/access-control#roles
382363
#
383364
# Empower Release Manager admins to create and manage repositories and

infra/gcp/infra.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,6 @@ infra:
283283
release:
284284
managed_by: infra/gcp/bash/ensure-release-projects.sh
285285
projects:
286-
k8s-release:
287286
k8s-release-test-prod: # TODO: should this be prod or release?:
288287

289288
releng:
@@ -356,7 +355,6 @@ infra:
356355
k8s-staging-kubeadm:
357356
k8s-staging-kubebuilder:
358357
k8s-staging-kueue:
359-
k8s-staging-kubernetes:
360358
k8s-staging-kubetest2:
361359
k8s-staging-kustomize:
362360
k8s-staging-kwok:

infra/gcp/terraform/k8s-infra-releases-prod/.terraform.lock.hcl

Lines changed: 62 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

infra/gcp/terraform/k8s-infra-releases-prod/iam.tf

Lines changed: 16 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,43 +14,6 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
data "google_iam_policy" "releng_access" {
18-
binding {
19-
role = "roles/storage.objectViewer"
20-
members = [
21-
"group:k8s-infra-release-editors@kubernetes.io",
22-
"serviceAccount:${google_service_account.fastly_reader.email}"
23-
]
24-
}
25-
26-
// TODO: remove this after https://github.com/kubernetes/release/issues/3425
27-
binding {
28-
role = "roles/storage.objectAdmin"
29-
members = ["serviceAccount:648026197307@cloudbuild.gserviceaccount.com"]
30-
}
31-
32-
binding {
33-
role = "roles/storage.legacyBucketOwner"
34-
members = [
35-
"projectOwner:${google_project.project.project_id}",
36-
"projectEditor:${google_project.project.project_id}"
37-
]
38-
}
39-
40-
binding {
41-
role = "roles/storage.legacyBucketReader"
42-
members = [
43-
"projectViewer:${google_project.project.project_id}",
44-
"group:k8s-infra-release-editors@kubernetes.io"
45-
]
46-
}
47-
}
48-
49-
resource "google_storage_bucket_iam_policy" "releng_access_policy" {
50-
bucket = module.k8s_releases_prod.bucket_name
51-
policy_data = data.google_iam_policy.releng_access.policy_data
52-
}
53-
5417
/*
5518
Ensure audit logging is enabled for GCS.
5619
See: https://cloud.google.com/storage/docs/audit-logging
@@ -59,7 +22,7 @@ module "audit_log_config" {
5922
source = "terraform-google-modules/iam/google//modules/audit_config"
6023
version = "~> 8.1"
6124

62-
project = google_project.project.project_id
25+
project = module.project.project_id
6326

6427
audit_log_config = [
6528
{
@@ -69,3 +32,18 @@ module "audit_log_config" {
6932
}
7033
]
7134
}
35+
36+
module "iam" {
37+
source = "terraform-google-modules/iam/google//modules/projects_iam"
38+
version = "~> 8"
39+
40+
projects = [module.project.project_id]
41+
42+
mode = "authoritative"
43+
44+
bindings = {
45+
"roles/viewer" = [
46+
"group:k8s-infra-release-editors@kubernetes.io",
47+
]
48+
}
49+
}

infra/gcp/terraform/k8s-infra-releases-prod/main.tf

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,35 +14,54 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
locals {
18-
billing_account = "018801-93540E-22A20E"
19-
org_id = "758905017065"
17+
module "project" {
18+
source = "terraform-google-modules/project-factory/google"
19+
version = "~> 18.2"
20+
21+
name = "k8s-infra-releases-prod"
2022
project_id = "k8s-infra-releases-prod"
21-
}
23+
folder_id = "455406320404" # Release Engineering
24+
billing_account = "018801-93540E-22A20E"
2225

26+
# Sane project defaults
27+
default_service_account = "keep"
28+
disable_services_on_destroy = false
29+
create_project_sa = false
30+
random_project_id = false
31+
auto_create_network = true
2332

24-
resource "google_project" "project" {
25-
name = local.project_id
26-
project_id = local.project_id
27-
org_id = local.org_id
28-
billing_account = local.billing_account
29-
auto_create_network = false
30-
}
3133

32-
module "k8s_releases_prod" {
33-
source = "../modules/k8s-releases"
34-
project_id = google_project.project.project_id
35-
bucket_name = "767373bbdcb8270361b96548387bf2a9ad0d48758c35"
34+
activate_apis = [
35+
"secretmanager.googleapis.com",
36+
"storage.googleapis.com",
37+
"storagetransfer.googleapis.com",
38+
]
3639
}
3740

3841
resource "google_service_account" "fastly_reader" {
39-
project = google_project.project.project_id
42+
project = module.project.project_id
4043
account_id = "fastly-reader"
4144
description = "Used by Fastly for read-only actions against the bucket"
4245
}
4346

4447
resource "google_storage_hmac_key" "fastly_reader_key" {
45-
project = google_project.project.project_id
48+
project = module.project.project_id
4649
service_account_email = google_service_account.fastly_reader.email
4750
}
4851

52+
53+
module "release_bucket" {
54+
source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
55+
version = "~> 12.3"
56+
57+
name = "767373bbdcb8270361b96548387bf2a9ad0d48758c35"
58+
project_id = module.project.project_id
59+
location = "us-central1"
60+
61+
iam_members = [
62+
{
63+
role = "roles/storage.objectAdmin"
64+
member = "serviceAccount:648026197307@cloudbuild.gserviceaccount.com"
65+
}
66+
]
67+
}

infra/gcp/terraform/k8s-infra-releases-prod/providers.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ This file defines:
2121
*/
2222

2323
terraform {
24-
required_version = "1.12"
24+
required_version = "1.12.2"
2525
backend "gcs" {
2626
bucket = "k8s-infra-tf-k8s-releases"
2727
prefix = "prod/terraform.tfstate"

infra/gcp/terraform/k8s-infra-releases-prod/secrets.tf

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,9 @@ module "secrets" {
1818
source = "GoogleCloudPlatform/secret-manager/google"
1919
version = "~> 0.9"
2020

21-
project_id = google_project.project.project_id
21+
project_id = module.project.project_id
2222

2323
secrets = [
24-
{
25-
name = "datadog_fastly_logs_streaming"
26-
secret_data = "REDACT-ME"
27-
},
2824
{
2925
name = "fastly_reader_sa_access_key"
3026
secret_data = google_storage_hmac_key.fastly_reader_key.access_id

0 commit comments

Comments
 (0)