✨ Add end-to-end testing

.gitignore

@@ -1,6 +1,8 @@
 /_build/
 /_tools/
 /vendor/
+/.kcp.e2e/
+/.e2e/
 .cover
 *.kubeconfig
 *.pem

.golangci.yml

@@ -43,9 +43,9 @@ linters:
     - predeclared
     - promlinter
     - staticcheck
-    - tenv
     - unconvert
     - unused
+    - usetesting
     - wastedassign
     - whitespace
   disable-all: true

.prow.yaml

@@ -87,3 +87,22 @@ presubmits:
             requests:
               memory: 4Gi
               cpu: 2
+
+  - name: pull-api-syncagent-test-e2e
+    always_run: true
+    decorate: true
+    clone_uri: "https://github.com/kcp-dev/api-syncagent"
+    labels:
+      preset-goproxy: "true"
+    spec:
+      containers:
+        - image: ghcr.io/kcp-dev/infra/build:1.23.5-1
+          command:
+            - hack/ci/run-e2e-tests.sh
+          resources:
+            requests:
+              memory: 4Gi
+              cpu: 2
+          # docker-in-docker needs privileged mode
+          securityContext:
+            privileged: true

Makefile

@@ -40,7 +40,7 @@ $(BUILD_DEST)/%: cmd/%
 	go build $(GOTOOLFLAGS) -o $@ ./cmd/$*
 
 GOLANGCI_LINT = _tools/golangci-lint
-GOLANGCI_LINT_VERSION = 1.63.4
+GOLANGCI_LINT_VERSION = 1.64.2
 
 .PHONY: $(GOLANGCI_LINT)
 $(GOLANGCI_LINT):
@@ -91,6 +91,23 @@ $(YQ):
 	  ${YQ_VERSION} \
 	  yq_*
 
+KCP = _tools/kcp
+KCP_VERSION = 0.26.1
+
+.PHONY: $(KCP)
+$(KCP):
+	@hack/download-tool.sh \
+	  https://github.com/kcp-dev/kcp/releases/download/v${KCP_VERSION}/kcp_${KCP_VERSION}_${GOOS}_${GOARCH}.tar.gz \
+	  kcp \
+	  ${KCP_VERSION}
+
+ENVTEST = _tools/setup-envtest
+ENVTEST_VERSION = release-0.19
+
+.PHONY: $(ENVTEST)
+$(ENVTEST):
+	@GO_MODULE=true hack/download-tool.sh sigs.k8s.io/controller-runtime/tools/setup-envtest setup-envtest $(ENVTEST_VERSION)
+
 .PHONY: test
 test:
 	./hack/run-tests.sh

go.mod

@@ -5,6 +5,7 @@ go 1.22.0
 require (
 	github.com/Masterminds/sprig/v3 v3.3.0
 	github.com/evanphx/json-patch/v5 v5.9.0
+	github.com/go-logr/logr v1.4.2
 	github.com/go-logr/zapr v1.3.0
 	github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20240817110845-a9eb9752bfeb
 	github.com/kcp-dev/client-go v0.0.0-20240912145314-f5949d81732a
@@ -44,7 +45,6 @@ require (
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect

hack/ci/run-e2e-tests.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The KCP Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+source hack/lib.sh
+
+# have a place to store things
+if [ -z "${ARTIFACTS:-}" ]; then
+  ARTIFACTS=.e2e/artifacts
+  mkdir -p "$ARTIFACTS"
+fi
+
+echodate "Build artifacts will be placed in $ARTIFACTS."
+export ARTIFACTS="$(realpath "$ARTIFACTS")"
+
+# build the agent, we will start it many times during the tests
+echodate "Building the api-syncagent…"
+make build
+
+# get kube envtest binaries
+echodate "Setting up Kube binaries…"
+make _tools/setup-envtest
+export KUBEBUILDER_ASSETS="$(_tools/setup-envtest use 1.31.0 --bin-dir _tools -p path)"
+KUBEBUILDER_ASSETS="$(realpath "$KUBEBUILDER_ASSETS")"
+
+# start a shared kcp process
+make _tools/kcp
+
+KCP_ROOT_DIRECTORY=.kcp.e2e
+KCP_LOGFILE="$ARTIFACTS/kcp.log"
+KCP_TOKENFILE=hack/ci/testdata/e2e-kcp.tokens
+
+echodate "Starting kcp…"
+rm -rf "$KCP_ROOT_DIRECTORY" "$KCP_LOGFILE"
+_tools/kcp start \
+  -v4 \
+  --token-auth-file "$KCP_TOKENFILE" \
+  --root-directory "$KCP_ROOT_DIRECTORY" 1>"$KCP_LOGFILE" 2>&1 &
+
+stop_kcp() {
+  echodate "Stopping kcp processes (set \$KEEP_KCP=true to not do this)…"
+  pkill -e kcp
+}
+
+if [[ -v KEEP_KCP ]] && $KEEP_KCP; then
+  echodate "\$KEEP_KCP is set, will not stop kcp once the script is finished."
+else
+  append_trap stop_kcp EXIT
+fi
+
+# make the token available to the Go tests
+export KCP_AGENT_TOKEN="$(grep e2e "$KCP_TOKENFILE" | cut -f1 -d,)"
+
+# Wait for kcp to be ready; this env name is also hardcoded in the Go tests.
+export KCP_KUBECONFIG="$KCP_ROOT_DIRECTORY/admin.kubeconfig"
+
+# the tenancy API becomes available pretty late during startup, so it's a good readiness check
+if ! retry_linear 3 20 kubectl --kubeconfig "$KCP_KUBECONFIG" get workspaces; then
+  echodate "kcp never became ready."
+  exit 1
+fi
+
+# makes it easier to reference thesefiles from various _test.go files.
+export ROOT_DIRECTORY="$(realpath .)"
+export KCP_KUBECONFIG="$(realpath "$KCP_KUBECONFIG")"
+export AGENT_BINARY="$(realpath _build/api-syncagent)"
+
+# time to run the tests
+echodate "Running e2e tests…"
+(set -x; go test -tags e2e -timeout 2h -v ./test/e2e/...)
+
+echodate "Done. :-)"

hack/ci/testdata/e2e-kcp.tokens

@@ -0,0 +1 @@
+topphemmelig,api-syncagent-e2e,1111-2222-3333-4444,"api-syncagents"

hack/download-tool.sh

@@ -17,7 +17,6 @@
 set -euo pipefail
 
 cd $(dirname $0)/..
-source hack/lib.sh
 
 mkdir -p _tools
 cd _tools
@@ -26,6 +25,8 @@ URL="$1"
 BINARY="$2"
 VERSION="$3"
 BINARY_PATTERN="${4:-**/$BINARY}"
+GO_MODULE=${GO_MODULE:-false}
+UNCOMPRESSED=${UNCOMPRESSED:-false}
 
 # Check if and what version we installed already.
 versionFile="$BINARY.version"
@@ -45,27 +46,31 @@ fi
   cd tmp
 
   echo "Downloading $BINARY version $VERSION …" >&2
-  curl --fail --silent -LO "$URL"
-  archive="$(ls)"
 
-  UNCOMPRESSED=${UNCOMPRESSED:-false}
+  if $GO_MODULE; then
+    GOBIN=$(realpath .) go install "$URL@$VERSION"
+    mv * "../$BINARY"
+  else
+    curl --fail --silent -LO "$URL"
+    archive="$(ls)"
 
-  if ! $UNCOMPRESSED; then
-    case "$archive" in
-      *.tar.gz | *.tgz)
-        tar xzf "$archive"
-        ;;
-      *.zip)
-        unzip "$archive"
-        ;;
-      *)
-        echo "Unknown file type: $archive" >&2
-        exit 1
-    esac
-  fi
+    if ! $UNCOMPRESSED; then
+      case "$archive" in
+        *.tar.gz | *.tgz)
+          tar xzf "$archive"
+          ;;
+        *.zip)
+          unzip "$archive"
+          ;;
+        *)
+          echo "Unknown file type: $archive" >&2
+          exit 1
+      esac
+    fi
 
-  mv $BINARY_PATTERN ../$BINARY
-  chmod +x ../$BINARY
+    mv $BINARY_PATTERN ../$BINARY
+    chmod +x ../$BINARY
+  fi
 )
 
 rm -rf tmp

hack/lib.sh

@@ -36,16 +36,16 @@ retry() {
   # Works only with bash but doesn't fail on other shells
   start_time=$(date +%s)
   set +e
-  actual_retry $@
+  # We use an extra wrapping to write junit and have a timer
+  retry_backoff $@
   rc=$?
   set -e
   elapsed_time=$(($(date +%s) - $start_time))
   write_junit "$rc" "$elapsed_time"
   return $rc
 }
 
-# We use an extra wrapping to write junit and have a timer
-actual_retry() {
+retry_backoff() {
   retries=$1
   shift
 
@@ -66,9 +66,33 @@ actual_retry() {
   return 0
 }
 
+retry_linear() {
+  delay=$1
+  retries=$2
+  shift
+  shift
+
+  count=0
+  until "$@"; do
+    rc=$?
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      echodate "[$count/$retries] Command returned $rc, retrying…"
+      sleep $delay
+    else
+      echodate "Command returned $rc, no more retries left."
+      return $rc
+    fi
+  done
+
+  echodate "Command succeeded."
+
+  return 0
+}
+
 echodate() {
   # do not use -Is to keep this compatible with macOS
-  echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@"
+  echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" > /dev/stderr
 }
 
 write_junit() {

hack/verify-boilerplate.sh

@@ -28,4 +28,5 @@ _tools/boilerplate \
   -exclude internal/certificates/triple \
   -exclude sdk/clientset \
   -exclude sdk/informers \
-  -exclude sdk/listers
+  -exclude sdk/listers \
+  -exclude test/crds

test/crds/crontab.yaml

@@ -0,0 +1,31 @@
+# sourced from https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: crontabs.example.com
+spec:
+  group: example.com
+  scope: Namespaced
+  names:
+    plural: crontabs
+    singular: crontab
+    kind: CronTab
+    shortNames:
+      - ct
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                cronSpec:
+                  type: string
+                image:
+                  type: string
+                replicas:
+                  type: integer