Merge pull request #485 from danilo-gemoli/fix/restore-hack-tools

Restore hack/tools module

Author: k8s-ci-robot

go.mod

@@ -121,7 +121,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/protobuf v1.5.4
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/cel-go v0.20.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
@@ -191,6 +191,13 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.3
+	github.com/client9/misspell v0.3.4
+	github.com/go-bindata/go-bindata/v3 v3.1.3
+	github.com/golangci/golangci-lint v1.64.8
+	github.com/google/ko v0.14.1
+	gotest.tools/gotestsum v1.12.3
+	k8s.io/code-generator v0.30.1
+	sigs.k8s.io/controller-tools v0.15.0
 )
 
 require (
@@ -262,7 +269,6 @@ require (
 	github.com/chavacava/garif v0.1.0 // indirect
 	github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
 	github.com/ckaznocha/intrange v0.3.0 // indirect
-	github.com/client9/misspell v0.3.4 // indirect
 	github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/curioswitch/go-reassign v0.3.0 // indirect
@@ -286,7 +292,6 @@ require (
 	github.com/firefart/nonamedreturns v1.0.5 // indirect
 	github.com/fzipp/gocyclo v0.6.0 // indirect
 	github.com/ghostiam/protogetter v0.3.9 // indirect
-	github.com/go-bindata/go-bindata/v3 v3.1.3 // indirect
 	github.com/go-critic/go-critic v0.12.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
@@ -312,13 +317,11 @@ require (
 	github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect
 	github.com/golangci/go-printf-func-name v0.1.0 // indirect
 	github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect
-	github.com/golangci/golangci-lint v1.64.8 // indirect
 	github.com/golangci/misspell v0.6.0 // indirect
 	github.com/golangci/plugin-module-register v0.1.1 // indirect
 	github.com/golangci/revgrep v0.8.0 // indirect
 	github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed // indirect
 	github.com/google/go-containerregistry v0.19.2 // indirect
-	github.com/google/ko v0.14.1 // indirect
 	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gordonklaus/ineffassign v0.1.0 // indirect
@@ -437,13 +440,10 @@ require (
 	golang.org/x/exp/typeparams v0.0.0-20250210185358-939b2ce775ac // indirect
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 // indirect
 	gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
-	gotest.tools/gotestsum v1.12.3 // indirect
 	honnef.co/go/tools v0.6.1 // indirect
-	k8s.io/code-generator v0.30.1 // indirect
 	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
 	mvdan.cc/gofumpt v0.7.0 // indirect
 	mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect
-	sigs.k8s.io/controller-tools v0.15.0 // indirect
 	sigs.k8s.io/kind v0.20.0 // indirect
 )
 

hack/make-rules/go-run/arbitrary.sh

@@ -26,6 +26,12 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
 cd "${REPO_ROOT}"
 source hack/build/setup-go.sh
 
+# build gotestsum
+cd 'hack/tools'
+go build -o "${REPO_ROOT}/_bin/gotestsum" gotest.tools/gotestsum
+# Make sure the following is ran from root dir
+cd "${REPO_ROOT}"
+
 (
   set -x;
   go "$@"

hack/make-rules/go-test/unit.sh

@@ -47,6 +47,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
 cd "${REPO_ROOT}"
 source hack/build/setup-go.sh
 
+# build gotestsum
+cd 'hack/tools'
+go build -o "${REPO_ROOT}/_bin/gotestsum" gotest.tools/gotestsum
+cd "${REPO_ROOT}"
+
 JUNIT_RESULT_DIR="${REPO_ROOT}/_output"
 # if we are in CI, copy to the artifact upload location
 if [[ -n "${ARTIFACTS:-}" ]]; then
@@ -58,7 +63,7 @@ fi
   set -x;
   umask 0022
   mkdir -p "${JUNIT_RESULT_DIR}"
-  go tool gotestsum --junitfile="${JUNIT_RESULT_DIR}/junit-unit.xml" \
+  "${REPO_ROOT}/_bin/gotestsum" --junitfile="${JUNIT_RESULT_DIR}/junit-unit.xml" \
     -- \
     ${PROW_UNIT_TEST_EXTRA_FLAGS[@]+${PROW_UNIT_TEST_EXTRA_FLAGS[@]}} \
     "./${folder_to_test}"

hack/make-rules/update/codegen.sh

@@ -33,17 +33,26 @@ cd $REPO_ROOT
 echo "Ensuring go version."
 source ./hack/build/setup-go.sh
 
-wrap-go-tool() {
-  local tool="$1"
-  local wrapper_name="${tool}-XXXX"
-  wrapper="$(mktemp --tmpdir $wrapper_name)"
-  printf '#!/usr/bin/bash\n\ngo tool %s $@\n' "$tool" >>"$wrapper"
-  chmod +x "$wrapper"
-  echo "$wrapper"
-}
-PROTOC_GEN_GO_WRAPPER="$(wrap-go-tool protoc-gen-go)"
-PROTOC_GEN_GO_GRPC_WRAPPER="$(wrap-go-tool protoc-gen-go-grpc)"
-
+# build codegen tools
+echo "Install codegen tools."
+cd "hack/tools"
+clientgen=${REPO_ROOT}/_bin/client-gen
+go build -o "${clientgen}" k8s.io/code-generator/cmd/client-gen
+deepcopygen=${REPO_ROOT}/_bin/deepcopy-gen
+go build -o "${deepcopygen}" k8s.io/code-generator/cmd/deepcopy-gen
+informergen=${REPO_ROOT}/_bin/informer-gen
+go build -o "${informergen}" k8s.io/code-generator/cmd/informer-gen
+listergen=${REPO_ROOT}/_bin/lister-gen
+go build -o "${listergen}" k8s.io/code-generator/cmd/lister-gen
+go_bindata=${REPO_ROOT}/_bin/go-bindata
+go build -o "${go_bindata}" github.com/go-bindata/go-bindata/v3/go-bindata
+controllergen=${REPO_ROOT}/_bin/controller-gen
+go build -o "${controllergen}" sigs.k8s.io/controller-tools/cmd/controller-gen
+protoc_gen_go="${REPO_ROOT}/_bin/protoc-gen-go" # golang protobuf plugin
+GOBIN="${REPO_ROOT}/_bin" go install google.golang.org/protobuf/cmd/[email protected]
+GOBIN="${REPO_ROOT}/_bin" go install google.golang.org/grpc/cmd/[email protected]
+
+cd "${REPO_ROOT}"
 ensure-protoc-deps() {
   # Install protoc
   if [[ ! -f "_bin/protoc/bin/protoc" ]]; then
@@ -88,15 +97,15 @@ gen-prow-config-documented() {
 
 gen-deepcopy() {
   echo "Generating DeepCopy() methods..." >&2
-  go tool deepcopy-gen ./... \
+  "$deepcopygen" ./... \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --output-file zz_generated.deepcopy.go \
     --bounding-dirs sigs.k8s.io/prow/pkg/apis,sigs.k8s.io/prow/pkg/config
 }
 
 gen-client() {
   echo "Generating client..." >&2
-  go tool client-gen \
+  "$clientgen" \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --clientset-name versioned \
     --input-base "" \
@@ -105,7 +114,7 @@ gen-client() {
     --output-pkg sigs.k8s.io/prow/pkg/client/clientset
 
   echo "Generating client for pipeline..." >&2
-  go tool client-gen \
+  "$clientgen" \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --clientset-name versioned \
     --input-base "" \
@@ -116,29 +125,29 @@ gen-client() {
 
 gen-lister() {
   echo "Generating lister..." >&2
-  go tool lister-gen sigs.k8s.io/prow/pkg/apis/prowjobs/v1 \
+  "$listergen" sigs.k8s.io/prow/pkg/apis/prowjobs/v1 \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --output-dir pkg/client/listers \
     --output-pkg sigs.k8s.io/prow/pkg/client/listers
 
   echo "Generating lister for pipeline..." >&2
-  go tool lister-gen github.com/tektoncd/pipeline/pkg/apis/pipeline/v1 \
+  "$listergen" github.com/tektoncd/pipeline/pkg/apis/pipeline/v1 \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --output-dir pkg/pipeline/listers \
     --output-pkg sigs.k8s.io/prow/pkg/pipeline/listers
 }
 
 gen-informer() {
   echo "Generating informer..." >&2
-  go tool informer-gen sigs.k8s.io/prow/pkg/apis/prowjobs/v1 \
+  "$informergen" sigs.k8s.io/prow/pkg/apis/prowjobs/v1 \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --versioned-clientset-package sigs.k8s.io/prow/pkg/client/clientset/versioned \
     --listers-package sigs.k8s.io/prow/pkg/client/listers \
     --output-dir pkg/client/informers \
     --output-pkg sigs.k8s.io/prow/pkg/client/informers
 
   echo "Generating informer for pipeline..." >&2
-  go tool informer-gen github.com/tektoncd/pipeline/pkg/apis/pipeline/v1 \
+  "$informergen" github.com/tektoncd/pipeline/pkg/apis/pipeline/v1 \
     --go-header-file hack/boilerplate/boilerplate.generated.go.txt \
     --versioned-clientset-package sigs.k8s.io/prow/pkg/pipeline/clientset/versioned \
     --listers-package sigs.k8s.io/prow/pkg/pipeline/listers \
@@ -149,14 +158,14 @@ gen-informer() {
 gen-spyglass-bindata() {
   cd pkg/spyglass/lenses/common/
   echo "Generating spyglass bindata..." >&2
-  go tool go-bindata -pkg=common static/
+  $go_bindata -pkg=common static/
   gofmt -s -w ./
   cd - >/dev/null
 }
 
 gen-prowjob-crd() {
   echo "Generating prowjob crd..." >&2
-  go tool controller-gen crd:crdVersions=v1 paths=./pkg/apis/prowjobs/v1 output:stdout \
+  "$controllergen" crd:crdVersions=v1 paths=./pkg/apis/prowjobs/v1 output:stdout \
     | $SED '/^$/d' \
     | $SED '/^spec:.*/a  \  preserveUnknownFields: false' \
     | $SED '/^  annotations.*/a  \    api-approved.kubernetes.io: https://github.com/kubernetes/test-infra/pull/8669' \
@@ -184,21 +193,22 @@ gen-proto-stubs() {
   # structure (so that the generated files can sit next to the .proto files,
   # instead of under a "k8.io/test-infra/prow/..." subfolder).
   "${REPO_ROOT}/_bin/protoc/bin/protoc" \
-    --plugin=protoc-gen-go="$PROTOC_GEN_GO_WRAPPER" \
-    --plugin=protoc-gen-go-grpc="$PROTOC_GEN_GO_GRPC_WRAPPER" \
-    --proto_path="${REPO_ROOT}/_bin/protoc/include/google/protobuf" \
-    --proto_path="${REPO_ROOT}/_bin/protoc/include/googleapis" \
-    --proto_path="$dir" \
-    --go_out="$dir" \
-    --go_opt=paths=source_relative \
-    --go-grpc_out="$dir" \
-    --go-grpc_opt=paths=source_relative \
+    "--plugin=${protoc_gen_go}" \
+    "--proto_path=${REPO_ROOT}/_bin/protoc/include/google/protobuf" \
+    "--proto_path=${REPO_ROOT}/_bin/protoc/include/googleapis" \
+    "--proto_path=${dir}" \
+    --go_out="${dir}" --go_opt=paths=source_relative \
+    --go-grpc_out="${dir}" --go-grpc_opt=paths=source_relative \
     "$1"
 }
 
 gen-all-proto-stubs() {
   echo >&2 "Generating proto stubs"
 
+  # Expose the golang protobuf plugin binaries (protoc-gen-go,
+  # protoc-gen-go-grpc) to the PATH so that protoc can find it.
+  export PATH="${REPO_ROOT}/_bin:$PATH"
+
   while IFS= read -r -d '' proto; do
     echo >&2 "  $proto"
     gen-proto-stubs "$proto"

hack/make-rules/update/misspell.sh

@@ -23,6 +23,13 @@ cd $REPO_ROOT
 echo "Ensuring go version."
 source ./hack/build/setup-go.sh
 
+# build misspell
+echo "Install misspell."
+cd "hack/tools"
+go build -o "${REPO_ROOT}/_bin/misspell" github.com/client9/misspell/cmd/misspell
+MISSPELL="${REPO_ROOT}/_bin/misspell"
+cd "${REPO_ROOT}"
+
 find -L . -type f -not \( \
   \( \
     -path '*/vendor/*' \
@@ -36,8 +43,8 @@ find -L . -type f -not \( \
     -o -path './_output/*' \
     -o -path './_artifacts/*' \
     -o -path './bazel-*/*' \
-    -o -path './go.mod' \
-    -o -path './go.sum' \
+    -o -path './hack/tools/go.mod' \
+    -o -path './hack/tools/go.sum' \
     -o -path './.python_virtual_env/*' \
     \) -prune \
-    \) | xargs go tool misspell -w
+    \) | xargs "${MISSPELL}" -w

hack/make-rules/verify/golangci-lint.sh

@@ -30,10 +30,16 @@ if [[ ! -f .golangci.yml ]]; then
   exit 1
 fi
 
+# build golangci-lint
+echo "Install golangci-lint."
+cd "hack/tools"
+go build -o "${REPO_ROOT}/_bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint
+cd "${REPO_ROOT}"
+
 echo "Run golangci-lint."
 echo "Go version: $(go version)"
-echo "Golangci-lint version: $(go tool golangci-lint version)"
+echo "Golangci-lint version: $(./_bin/golangci-lint version)"
 export GO111MODULE=on
 export GOPROXY=https://proxy.golang.org
 export GOSUMDB=sum.golang.org
-go tool golangci-lint run ./...
+./_bin/golangci-lint run ./...

hack/make-rules/verify/misspell.sh

@@ -20,9 +20,19 @@ set -o pipefail
 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
 cd $REPO_ROOT
 
+BIN_DIR="${REPO_ROOT}/_bin/misspell"
+mkdir -p "$BIN_DIR"
+MISSPELL="${BIN_DIR}/misspell"
+
 echo "Ensuring go version."
 source ./hack/build/setup-go.sh
 
+# build misspell
+echo "Install misspell."
+cd "hack/tools"
+go build -o "$MISSPELL" github.com/client9/misspell/cmd/misspell
+cd "${REPO_ROOT}"
+
 trap 'echo ERROR: found unexpected instance of "Git"hub, use github or GitHub' ERR
 
 echo "Check for word 'github'..."
@@ -68,10 +78,12 @@ find -L . -type f -not \( \
     -o -path './_output/*' \
     -o -path './_artifacts/*' \
     -o -path './bazel-*/*' \
+    -o -path './hack/tools/go.mod' \
+    -o -path './hack/tools/go.sum' \
     -o -path './go.mod' \
     -o -path './go.sum' \
     -o -path './.python_virtual_env/*' \
     \) -prune \
-    \) | xargs go tool misspell --error
+    \) | xargs "$MISSPELL" --error
 
 echo 'PASS: No spelling issues detected'

hack/prowimagebuilder/main.go

@@ -71,6 +71,11 @@ func init() {
 		os.Exit(1)
 	}
 	rootDir = out
+
+	if _, err := runCmdInDirFunc(path.Join(rootDir, "hack/tools"), nil, "go", "build", "-o", path.Join(rootDir, "_bin/ko"), "github.com/google/ko"); err != nil {
+		logrus.WithError(err).Error("Failed ensure ko")
+		os.Exit(1)
+	}
 }
 
 type options struct {
@@ -126,10 +131,6 @@ func runCmd(additionalEnv []string, cmd string, args ...string) (string, error)
 	return runCmdInDirFunc(rootDir, additionalEnv, cmd, args...)
 }
 
-func runGoToolCmd(additionalEnv []string, tool string, args ...string) (string, error) {
-	return runCmd(additionalEnv, "go", append([]string{"tool", tool}, args...)...)
-}
-
 type imageDef struct {
 	Dir            string `json:"dir"`
 	Arch           string `json:"arch"`
@@ -267,7 +268,7 @@ func buildAndPush(id *imageDef, dockerRepos []string, push bool) error {
 	// to subsequent identical docker repo(s) is relatively cheap.
 	for _, dockerRepo := range dockerRepos {
 		logger.WithField("args", publishArgs).Info("Running ko.")
-		if _, err = runGoToolCmd([]string{"KO_DOCKER_REPO=" + dockerRepo}, "ko", publishArgs...); err != nil {
+		if _, err = runCmd([]string{"KO_DOCKER_REPO=" + dockerRepo}, "_bin/ko", publishArgs...); err != nil {
 			return fmt.Errorf("running ko: %w", err)
 		}
 	}