Merge pull request #176 from sunnylovestiramisu/updateReleaseTools

Update release tools

Author: k8s-ci-robot

release-tools/.github/dependabot.yaml

@@ -0,0 +1,12 @@
+version: 2
+enable-beta-ecosystems: true
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+      interval: "daily"
+  labels:
+    - "area/dependency"
+    - "release-note-none"
+    - "ok-to-test"
+  open-pull-requests-limit: 10

release-tools/.github/workflows/codespell.yml

@@ -0,0 +1,15 @@
+# GitHub Action to automate the identification of common misspellings in text files.
+# https://github.com/codespell-project/actions-codespell
+# https://github.com/codespell-project/codespell
+name: codespell
+on: [push, pull_request]
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: codespell-project/actions-codespell@master
+        with:
+          check_filenames: true
+          skip: "*.png,*.jpg,*.svg,*.sum,./.git,./.github/workflows/codespell.yml,./prow.sh"

release-tools/.github/workflows/trivy.yaml

@@ -0,0 +1,29 @@
+name: Run Trivy scanner for Go version vulnerabilities
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  trivy:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Go version
+        id: go-version
+        run: |
+          GO_VERSION=$(cat prow.sh  | grep "configvar CSI_PROW_GO_VERSION_BUILD" | awk '{print $3}' | sed 's/"//g')
+          echo "version=$GO_VERSION" >> $GITHUB_OUTPUT
+
+      - name: Run Trivy scanner for Go version vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'golang:${{ steps.go-version.outputs.version }}'
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'library'
+          severity: 'CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN'

release-tools/KUBERNETES_CSI_OWNERS_ALIASES

@@ -18,6 +18,7 @@ aliases:
   # when they are temporarily unable to review PRs.
   kubernetes-csi-reviewers:
   - andyzhangx
+  - carlory
   - chrishenzie
   - ggriffiths
   - gnufied

release-tools/SIDECAR_RELEASE_PROCESS.md

@@ -17,7 +17,7 @@ The release manager must:
 Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs
 must be updated.
 
-[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the
+[Our CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) have the
 naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 
 1. Jobs should be actively monitored to find and fix failures in sidecars and
@@ -46,62 +46,59 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 ## Release Process
 1. Identify all issues and ongoing PRs that should go into the release, and
   drive them to resolution.
-1. Download the latest version of the
-   [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes)
-1. Create a
-   [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
-   with `repo:public_repo` access
-1. Generate release notes for the release. Replace arguments with the relevant
-   information.
-    * Clean up old cached information (also needed if you are generating release
-      notes for multiple repos)
-      ```bash
-      rm -rf /tmp/k8s-repo
-      ```
-    * For new minor releases on master:
-        ```bash
-        GITHUB_TOKEN=<token> release-notes \
-          --discover=mergebase-to-latest \
-          --org=kubernetes-csi \
-          --repo=external-provisioner \
-          --required-author="" \
-          --markdown-links \
-          --output out.md
-        ```
-    * For new patch releases on a release branch:
-        ```bash
-        GITHUB_TOKEN=<token> release-notes \
-          --discover=patch-to-latest \
-          --branch=release-1.1 \
-          --org=kubernetes-csi \
-          --repo=external-provisioner \
-          --required-author="" \
-          --markdown-links \
-          --output out.md
-        ```
-1. Compare the generated output to the new commits for the release to check if
-   any notable change missed a release note.
-1. Reword release notes as needed. Make sure to check notes for breaking
-   changes and deprecations.
-1. If release is a new major/minor version, create a new `CHANGELOG-<major>.<minor>.md`
-   file. Otherwise, add the release notes to the top of the existing CHANGELOG
-   file for that minor version.
-1. Submit a PR for the CHANGELOG changes.
-1. Submit a PR for README changes, in particular, Compatibility, Feature status,
-   and any other sections that may need updating.
+1. Update dependencies for sidecars
+    1. For new minor versions, use
+   [go-modules-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-update.sh),
+    1. For CVE fixes on patch versions, use
+   [go-modules-targeted-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-targeted-update.sh),
+       Read the instructions at the top of the script.
 1. Check that all [canary CI
-  jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing,
+  jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing,
   and that test coverage is adequate for the changes that are going into the release.
 1. Check that the post-\<sidecar\>-push-images builds are succeeding.
-   [Example](https://k8s-testgrid.appspot.com/sig-storage-image-build#post-external-snapshotter-push-images)
+   [Example](https://testgrid.k8s.io/sig-storage-image-build#post-external-snapshotter-push-images)
+1. Generate release notes.
+    1.  Download the latest version of the [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes)
+    1. Create a
+       [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
+       with `repo:public_repo` access
+    1. For patch release, use the script generate_patch_release_notes.sh. Read the instructions at the top of the
+       script. The script also creates PRs for each branch.
+    1. For new minor releases, follow these steps and replace arguments with the relevant
+       information.
+        * Clean up old cached information (also needed if you are generating release
+          notes for multiple repos)
+          ```bash
+          rm -rf /tmp/k8s-repo
+          ```
+        * For new minor releases on master:
+            ```bash
+            GITHUB_TOKEN=<token> release-notes \
+              --discover=mergebase-to-latest \
+              --org=kubernetes-csi \
+              --repo=external-provisioner \
+              --required-author="" \
+              --markdown-links \
+              --output out.md
+            ```
+    1. Compare the generated output to the new commits for the release to check if
+       any notable change missed a release note.
+    1. Reword release notes as needed, ideally in the original PRs so that the
+       release notes can be regenerated. Make sure to check notes for breaking
+       changes and deprecations.
+    1. If release is a new major/minor version, create a new `CHANGELOG-<major>.<minor>.md`
+       file.
+    1. Submit a PR for the CHANGELOG changes.
+1. Submit a PR for README changes, in particular, Compatibility, Feature status,
+   and any other sections that may need updating.
 1. Make sure that no new PRs have merged in the meantime, and no PRs are in
    flight and soon to be merged.
 1. Create a new release following a previous release as a template. Be sure to select the correct
    branch. This requires Github release permissions as required by the prerequisites.
    [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new)
 1. If release was a new major/minor version, create a new `release-<minor>`
    branch at that commit.
-1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
+1. Check [image build status](https://testgrid.k8s.io/sig-storage-image-build).
 1. Promote images from k8s-staging-sig-storage to registry.k8s.io/sig-storage. From
    the [k8s image
    repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage),
@@ -120,7 +117,7 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 
 The following jobs are triggered after tagging to produce the corresponding
 image(s):
-https://k8s-testgrid.appspot.com/sig-storage-image-build
+https://testgrid.k8s.io/sig-storage-image-build
 
 Clicking on a failed build job opens that job in https://prow.k8s.io. Next to
 the job title is a rerun icon (circle with arrow). Clicking it opens a popup

release-tools/build.make

@@ -45,9 +45,10 @@ REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git r
 # Determined dynamically.
 IMAGE_TAGS=
 
-# A "canary" image gets built if the current commit is the head of the remote "master" branch.
+# A "canary" image gets built if the current commit is the head of the remote "master" or "main" branch.
 # That branch does not exist when building some other branch in TravisCI.
 IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/master 2>/dev/null)" ]; then echo "canary"; fi)
+IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/main 2>/dev/null)" ]; then echo "canary"; fi)
 
 # A "X.Y.Z-canary" image gets built if the current commit is the head of a "origin/release-X.Y.Z" branch.
 # The actual suffix does not matter, only the "release-" prefix is checked.
@@ -62,9 +63,9 @@ IMAGE_NAME=$(REGISTRY_NAME)/$*
 
 ifdef V
 # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed.
-TESTARGS = -v -args -alsologtostderr -v 5
+TESTARGS = -race -v -args -alsologtostderr -v 5
 else
-TESTARGS =
+TESTARGS = -race
 endif
 
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
@@ -143,7 +144,7 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # Windows binaries can be built before adding a Dockerfile for it.
 #
 # BUILD_PLATFORMS determines which individual images are included in the multiarch image.
-# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name, and determines
 # the tag for the resulting multiarch image.
 $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
@@ -191,7 +192,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 		done; \
 		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
-	if [ $(PULL_BASE_REF) = "master" ]; then \
+	if [ $(PULL_BASE_REF) = "master" ] || [ $(PULL_BASE_REF) = "main" ]; then \
 			: "creating or overwriting canary image"; \
 			pushMultiArch canary; \
 	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
@@ -209,7 +210,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 .PHONY: check-pull-base-ref
 check-pull-base-ref:
 	if ! [ "$(PULL_BASE_REF)" ]; then \
-		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name."; \
 		exit 1; \
 	fi
 
@@ -322,3 +323,10 @@ test-spelling:
 test-boilerplate:
 	@ echo; echo "### $@:"
 	@ ./release-tools/verify-boilerplate.sh "$(pwd)"
+
+# Test klog usage. This test is optional and must be explicitly added to `test` target in the main Makefile:
+# test: test-logcheck
+.PHONY: test-logcheck
+test-logcheck:
+	@ echo; echo "### $@:"
+	@ ./release-tools/verify-logcheck.sh

release-tools/cloudbuild.yaml

@@ -26,7 +26,7 @@ steps:
   # The image must contain bash and curl. Ideally it should also contain
   # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20230623-56e06d7c18'
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20240718-5ef92b5c36'
     entrypoint: ./.cloudbuild.sh
     env:
     - GIT_TAG=${_GIT_TAG}

release-tools/generate-patch-release-notes.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Usage: generate_patch_release_notes.sh
+#
+# Generates and creates PRs for kubernetes-csi patch releases.
+#
+# Required environment variables
+# CSI_RELEASE_TOKEN: Github token needed for generating release notes
+# GITHUB_USER: Github username to create PRs with
+#
+# Required tools:
+# - gh
+# - release-notes (https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md)
+# 
+# Instructions:
+# 1. Install the required tools
+# 2. Login with "gh auth login"
+# 3. Copy this script to the kubernetes-csi directory (one directory above the repos)
+# 4. Update the repos and versions in the $releases array
+# 5. Set environment variables
+# 6. Run script from the kubernetes-csi directory
+#
+# Caveats:
+# - This script doesn't handle regenerating and updating existing PRs yet.
+#   It might work if you comment out the PR creation line
+
+set -e
+set -x
+
+releases=(
+#  "external-attacher 4.4.1"
+#  "external-provisioner 3.6.1"
+#  "external-snapshotter 6.2.3"
+)
+
+function gen_patch_relnotes() {
+  rm out.md || true
+  rm -rf /tmp/k8s-repo || true
+  GITHUB_TOKEN="$CSI_RELEASE_TOKEN" \
+  release-notes --start-rev="$3" --end-rev="$2" --branch="$2" \
+    --org=kubernetes-csi --repo="$1" \
+    --required-author="" --markdown-links --output out.md
+}
+
+for rel in "${releases[@]}"; do
+  read -r repo version <<< "$rel"
+
+  # Parse minor version
+  minorPatchPattern="(^[[:digit:]]+\.[[:digit:]]+)\.([[:digit:]]+)"
+  [[ "$version" =~ $minorPatchPattern ]]
+  minor="${BASH_REMATCH[1]}"
+  patch="${BASH_REMATCH[2]}"
+
+  echo "$repo $version $minor $patch"
+  prevPatch="$((patch-1))"
+  prevVer="v$minor.$prevPatch"
+
+  pushd "$repo/CHANGELOG"
+
+  git fetch upstream
+
+  # Create branch
+  branch="changelog-release-$minor"
+  git checkout master
+  git branch -D "$branch" || true
+  git checkout --track "upstream/release-$minor" -b "$branch"
+
+  # Generate release notes
+  gen_patch_relnotes "$repo" "release-$minor" "$prevVer"
+  cat > tmp.md <<EOF
+# Release notes for v$version
+
+[Documentation](https://kubernetes-csi.github.io)
+
+EOF
+
+  cat out.md >> tmp.md
+  echo >> tmp.md
+  rm out.md
+
+  file="CHANGELOG-$minor.md"
+  cat "$file" >> tmp.md
+  mv tmp.md "$file"
+
+  git add -u
+  git commit -m "Add changelog for $version"
+  git push -f origin "$branch"
+
+  # Create PR
+prbody=$(cat <<EOF
+\`\`\`release-note
+NONE
+\`\`\`
+EOF
+)
+  gh pr create --title="Changelog for v$version" --body "$prbody"  --head "$GITHUB_USER:$branch" --base "release-$minor" --repo="kubernetes-csi/$repo"
+
+  popd
+done