diff --git a/.gitignore b/.gitignore index e91c612..db758ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ /_build/ /_tools/ /vendor/ +/.kcp.e2e/ +/.e2e/ .cover *.kubeconfig *.pem diff --git a/.golangci.yml b/.golangci.yml index 314a8e4..f9c68cf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -43,9 +43,9 @@ linters: - predeclared - promlinter - staticcheck - - tenv - unconvert - unused + - usetesting - wastedassign - whitespace disable-all: true diff --git a/.prow.yaml b/.prow.yaml index 4e8017c..b08d5cf 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -87,3 +87,22 @@ presubmits: requests: memory: 4Gi cpu: 2 + + - name: pull-api-syncagent-test-e2e + always_run: true + decorate: true + clone_uri: "https://github.com/kcp-dev/api-syncagent" + labels: + preset-goproxy: "true" + spec: + containers: + - image: ghcr.io/kcp-dev/infra/build:1.23.5-1 + command: + - hack/ci/run-e2e-tests.sh + resources: + requests: + memory: 4Gi + cpu: 2 + # docker-in-docker needs privileged mode + securityContext: + privileged: true diff --git a/Makefile b/Makefile index 9adf104..504a8f3 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ $(BUILD_DEST)/%: cmd/% go build $(GOTOOLFLAGS) -o $@ ./cmd/$* GOLANGCI_LINT = _tools/golangci-lint -GOLANGCI_LINT_VERSION = 1.63.4 +GOLANGCI_LINT_VERSION = 1.64.2 .PHONY: $(GOLANGCI_LINT) $(GOLANGCI_LINT): @@ -91,6 +91,23 @@ $(YQ): ${YQ_VERSION} \ yq_* +KCP = _tools/kcp +KCP_VERSION = 0.26.1 + +.PHONY: $(KCP) +$(KCP): + @hack/download-tool.sh \ + https://github.com/kcp-dev/kcp/releases/download/v${KCP_VERSION}/kcp_${KCP_VERSION}_${GOOS}_${GOARCH}.tar.gz \ + kcp \ + ${KCP_VERSION} + +ENVTEST = _tools/setup-envtest +ENVTEST_VERSION = release-0.19 + +.PHONY: $(ENVTEST) +$(ENVTEST): + @GO_MODULE=true hack/download-tool.sh sigs.k8s.io/controller-runtime/tools/setup-envtest setup-envtest $(ENVTEST_VERSION) + .PHONY: test test: ./hack/run-tests.sh diff --git a/go.mod b/go.mod index d948c4d..aa312e0 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.0 require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/evanphx/json-patch/v5 v5.9.0 + github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20240817110845-a9eb9752bfeb github.com/kcp-dev/client-go v0.0.0-20240912145314-f5949d81732a @@ -44,7 +45,6 @@ require ( github.com/fatih/color v1.18.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh new file mode 100755 index 0000000..9d0640f --- /dev/null +++ b/hack/ci/run-e2e-tests.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# Copyright 2025 The KCP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +source hack/lib.sh + +# have a place to store things +if [ -z "${ARTIFACTS:-}" ]; then + ARTIFACTS=.e2e/artifacts + mkdir -p "$ARTIFACTS" +fi + +echodate "Build artifacts will be placed in $ARTIFACTS." +export ARTIFACTS="$(realpath "$ARTIFACTS")" + +# build the agent, we will start it many times during the tests +echodate "Building the api-syncagent…" +make build + +# get kube envtest binaries +echodate "Setting up Kube binaries…" +make _tools/setup-envtest +export KUBEBUILDER_ASSETS="$(_tools/setup-envtest use 1.31.0 --bin-dir _tools -p path)" +KUBEBUILDER_ASSETS="$(realpath "$KUBEBUILDER_ASSETS")" + +# start a shared kcp process +make _tools/kcp + +KCP_ROOT_DIRECTORY=.kcp.e2e +KCP_LOGFILE="$ARTIFACTS/kcp.log" +KCP_TOKENFILE=hack/ci/testdata/e2e-kcp.tokens + +echodate "Starting kcp…" +rm -rf "$KCP_ROOT_DIRECTORY" "$KCP_LOGFILE" +_tools/kcp start \ + -v4 \ + --token-auth-file "$KCP_TOKENFILE" \ + --root-directory "$KCP_ROOT_DIRECTORY" 1>"$KCP_LOGFILE" 2>&1 & + +stop_kcp() { + echodate "Stopping kcp processes (set \$KEEP_KCP=true to not do this)…" + pkill -e kcp +} + +if [[ -v KEEP_KCP ]] && $KEEP_KCP; then + echodate "\$KEEP_KCP is set, will not stop kcp once the script is finished." +else + append_trap stop_kcp EXIT +fi + +# make the token available to the Go tests +export KCP_AGENT_TOKEN="$(grep e2e "$KCP_TOKENFILE" | cut -f1 -d,)" + +# Wait for kcp to be ready; this env name is also hardcoded in the Go tests. +export KCP_KUBECONFIG="$KCP_ROOT_DIRECTORY/admin.kubeconfig" + +# the tenancy API becomes available pretty late during startup, so it's a good readiness check +if ! retry_linear 3 20 kubectl --kubeconfig "$KCP_KUBECONFIG" get workspaces; then + echodate "kcp never became ready." + exit 1 +fi + +# makes it easier to reference thesefiles from various _test.go files. +export ROOT_DIRECTORY="$(realpath .)" +export KCP_KUBECONFIG="$(realpath "$KCP_KUBECONFIG")" +export AGENT_BINARY="$(realpath _build/api-syncagent)" + +# time to run the tests +echodate "Running e2e tests…" +(set -x; go test -tags e2e -timeout 2h -v ./test/e2e/...) + +echodate "Done. :-)" diff --git a/hack/ci/testdata/e2e-kcp.tokens b/hack/ci/testdata/e2e-kcp.tokens new file mode 100644 index 0000000..bd8f688 --- /dev/null +++ b/hack/ci/testdata/e2e-kcp.tokens @@ -0,0 +1 @@ +topphemmelig,api-syncagent-e2e,1111-2222-3333-4444,"api-syncagents" diff --git a/hack/download-tool.sh b/hack/download-tool.sh index 12d9c61..44dc548 100755 --- a/hack/download-tool.sh +++ b/hack/download-tool.sh @@ -17,7 +17,6 @@ set -euo pipefail cd $(dirname $0)/.. -source hack/lib.sh mkdir -p _tools cd _tools @@ -26,6 +25,8 @@ URL="$1" BINARY="$2" VERSION="$3" BINARY_PATTERN="${4:-**/$BINARY}" +GO_MODULE=${GO_MODULE:-false} +UNCOMPRESSED=${UNCOMPRESSED:-false} # Check if and what version we installed already. versionFile="$BINARY.version" @@ -45,27 +46,31 @@ fi cd tmp echo "Downloading $BINARY version $VERSION …" >&2 - curl --fail --silent -LO "$URL" - archive="$(ls)" - UNCOMPRESSED=${UNCOMPRESSED:-false} + if $GO_MODULE; then + GOBIN=$(realpath .) go install "$URL@$VERSION" + mv * "../$BINARY" + else + curl --fail --silent -LO "$URL" + archive="$(ls)" - if ! $UNCOMPRESSED; then - case "$archive" in - *.tar.gz | *.tgz) - tar xzf "$archive" - ;; - *.zip) - unzip "$archive" - ;; - *) - echo "Unknown file type: $archive" >&2 - exit 1 - esac - fi + if ! $UNCOMPRESSED; then + case "$archive" in + *.tar.gz | *.tgz) + tar xzf "$archive" + ;; + *.zip) + unzip "$archive" + ;; + *) + echo "Unknown file type: $archive" >&2 + exit 1 + esac + fi - mv $BINARY_PATTERN ../$BINARY - chmod +x ../$BINARY + mv $BINARY_PATTERN ../$BINARY + chmod +x ../$BINARY + fi ) rm -rf tmp diff --git a/hack/lib.sh b/hack/lib.sh index c51eec8..cad4d66 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -36,7 +36,8 @@ retry() { # Works only with bash but doesn't fail on other shells start_time=$(date +%s) set +e - actual_retry $@ + # We use an extra wrapping to write junit and have a timer + retry_backoff $@ rc=$? set -e elapsed_time=$(($(date +%s) - $start_time)) @@ -44,8 +45,7 @@ retry() { return $rc } -# We use an extra wrapping to write junit and have a timer -actual_retry() { +retry_backoff() { retries=$1 shift @@ -66,9 +66,33 @@ actual_retry() { return 0 } +retry_linear() { + delay=$1 + retries=$2 + shift + shift + + count=0 + until "$@"; do + rc=$? + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + echodate "[$count/$retries] Command returned $rc, retrying…" + sleep $delay + else + echodate "Command returned $rc, no more retries left." + return $rc + fi + done + + echodate "Command succeeded." + + return 0 +} + echodate() { # do not use -Is to keep this compatible with macOS - echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" + echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" > /dev/stderr } write_junit() { diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index 7e15fb3..6f0f001 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -28,4 +28,5 @@ _tools/boilerplate \ -exclude internal/certificates/triple \ -exclude sdk/clientset \ -exclude sdk/informers \ - -exclude sdk/listers + -exclude sdk/listers \ + -exclude test/crds diff --git a/test/crds/crontab.yaml b/test/crds/crontab.yaml new file mode 100644 index 0000000..5ec5299 --- /dev/null +++ b/test/crds/crontab.yaml @@ -0,0 +1,31 @@ +# sourced from https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: crontabs.example.com +spec: + group: example.com + scope: Namespaced + names: + plural: crontabs + singular: crontab + kind: CronTab + shortNames: + - ct + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + cronSpec: + type: string + image: + type: string + replicas: + type: integer diff --git a/test/e2e/apiresourceschema/apiresourceschema_test.go b/test/e2e/apiresourceschema/apiresourceschema_test.go new file mode 100644 index 0000000..f55f7a0 --- /dev/null +++ b/test/e2e/apiresourceschema/apiresourceschema_test.go @@ -0,0 +1,108 @@ +//go:build e2e + +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiresourceschema + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + "github.com/kcp-dev/api-syncagent/test/utils" + + kcpapisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime" +) + +func TestARSAreCreated(t *testing.T) { + const ( + apiExportName = "example.com" + ) + + ctx := context.Background() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubconfig := utils.CreateOrganization(t, ctx, "ars-are-created", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab.yaml", + }) + + // publish Crontabs + t.Logf("Publishing CronTabs…") + pr := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: "v1", + Kind: "CronTab", + }, + }, + } + + if err := envtestClient.Create(ctx, pr); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // let the agent do its thing + utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName) + + // wait for the APIExport to be updated + t.Logf("Waiting for APIExport to be updated…") + orgClient := utils.GetClient(t, orgKubconfig) + apiExportKey := types.NamespacedName{Name: apiExportName} + + var arsName string + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (done bool, err error) { + apiExport := &kcpapisv1alpha1.APIExport{} + err = orgClient.Get(ctx, apiExportKey, apiExport) + if err != nil { + return false, err + } + + if len(apiExport.Spec.LatestResourceSchemas) == 0 { + return false, nil + } + + arsName = apiExport.Spec.LatestResourceSchemas[0] + + return true, nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport to be updated: %v", err) + } + + // check the APIResourceSchema + ars := &kcpapisv1alpha1.APIResourceSchema{} + err = orgClient.Get(ctx, types.NamespacedName{Name: arsName}, ars) + if err != nil { + t.Fatalf("APIResourceSchema does not exist: %v", err) + } +} diff --git a/test/utils/fixtures.go b/test/utils/fixtures.go new file mode 100644 index 0000000..8929411 --- /dev/null +++ b/test/utils/fixtures.go @@ -0,0 +1,303 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/kcp-dev/logicalcluster/v3" + + kcpapisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + kcpcorev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + kcptenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/kontext" +) + +func CreateOrganization( + t *testing.T, + ctx context.Context, + workspaceName logicalcluster.Name, + apiExportName string, +) string { + t.Helper() + + kcpClient := GetKcpAdminClusterClient(t) + agent := rbacv1.Subject{ + Kind: "User", + Name: "api-syncagent-e2e", + } + + // setup workspaces + orgClusterName := CreateWorkspace(t, ctx, kcpClient, "root", workspaceName) + + // grant access and allow the agent to resolve its own workspace path + homeCtx := kontext.WithCluster(ctx, orgClusterName) + GrantWorkspaceAccess(t, homeCtx, kcpClient, string(workspaceName), agent, rbacv1.PolicyRule{ + APIGroups: []string{"core.kcp.io"}, + Resources: []string{"logicalclusters"}, + ResourceNames: []string{"cluster"}, + Verbs: []string{"get"}, + }) + + // add some consumer workspaces + teamClusters := []logicalcluster.Name{ + CreateWorkspace(t, ctx, kcpClient, orgClusterName, "team-1"), + CreateWorkspace(t, ctx, kcpClient, orgClusterName, "team-2"), + } + + // setup the APIExport and wait for it to be ready + apiExport := CreateAPIExport(t, homeCtx, kcpClient, apiExportName, &agent) + + // bind it in all team workspaces, so the virtual workspace is ready inside kcp + for _, teamCluster := range teamClusters { + teamCtx := kontext.WithCluster(ctx, teamCluster) + BindToAPIExport(t, teamCtx, kcpClient, apiExport) + } + + return CreateKcpAgentKubeconfig(t, fmt.Sprintf("/clusters/%s", orgClusterName)) +} + +func CreateWorkspace(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, parent logicalcluster.Name, workspaceName logicalcluster.Name) logicalcluster.Name { + t.Helper() + + testWs := &kcptenancyv1alpha1.Workspace{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspaceName.String(), + }, + } + + ctx = kontext.WithCluster(ctx, parent) + + t.Logf("Creating workspace %s:%s…", parent, workspaceName) + if err := client.Create(ctx, testWs); err != nil { + t.Fatalf("Failed to create %q workspace: %v", workspaceName, err) + } + + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (done bool, err error) { + err = client.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(testWs), testWs) + if err != nil { + return false, err + } + + return testWs.Status.Phase == kcpcorev1alpha1.LogicalClusterPhaseReady, nil + }) + if err != nil { + t.Fatalf("Failed to wait for workspace to become ready: %v", err) + } + + return logicalcluster.Name(testWs.Spec.Cluster) +} + +func CreateAPIExport(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, name string, rbacSubject *rbacv1.Subject) *kcpapisv1alpha1.APIExport { + t.Helper() + + // create the APIExport to server with the Sync Agent + apiExport := &kcpapisv1alpha1.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + + t.Logf("Creating APIExport %q…", name) + if err := client.Create(ctx, apiExport); err != nil { + t.Fatalf("Failed to create APIExport: %v", err) + } + + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (done bool, err error) { + err = client.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(apiExport), apiExport) + if err != nil { + return false, err + } + + return conditions.IsTrue(apiExport, kcpapisv1alpha1.APIExportVirtualWorkspaceURLsReady), nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIExport virtual workspace to become ready: %v", err) + } + + // grant permissions to access/manage the APIExport + if rbacSubject != nil { + clusterRoleName := "api-syncagent" + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"apis.kcp.io"}, + Resources: []string{"apiexports"}, + ResourceNames: []string{name}, + Verbs: []string{"get", "list", "watch", "patch", "update"}, + }, + { + APIGroups: []string{"apis.kcp.io"}, + Resources: []string{"apiresourceschemas"}, + Verbs: []string{"get", "list", "watch", "create"}, + }, + { + APIGroups: []string{"apis.kcp.io"}, + Resources: []string{"apiexports/content"}, + ResourceNames: []string{name}, + Verbs: []string{"*"}, + }, + }, + } + + if err := client.Create(ctx, clusterRole); err != nil { + t.Fatalf("Failed to create ClusterRole: %v", err) + } + + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Subjects: []rbacv1.Subject{*rbacSubject}, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: clusterRoleName, + }, + } + + if err := client.Create(ctx, clusterRoleBinding); err != nil { + t.Fatalf("Failed to create ClusterRoleBinding: %v", err) + } + } + + return apiExport +} + +func GrantWorkspaceAccess(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, workspaceName string, rbacSubject rbacv1.Subject, extraRules ...rbacv1.PolicyRule) { + t.Helper() + + clusterRoleName := fmt.Sprintf("access-workspace:%s", strings.ToLower(rbacSubject.Name)) + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Rules: append([]rbacv1.PolicyRule{ + { + Verbs: []string{"access"}, + NonResourceURLs: []string{"/"}, + }, + }, extraRules...), + } + + if err := client.Create(ctx, clusterRole); err != nil { + t.Fatalf("Failed to create ClusterRole: %v", err) + } + + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "workspace-access-", + }, + Subjects: []rbacv1.Subject{rbacSubject}, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: clusterRoleName, + }, + } + + if err := client.Create(ctx, clusterRoleBinding); err != nil { + t.Fatalf("Failed to create ClusterRoleBinding: %v", err) + } +} + +func BindToAPIExport(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, apiExport *kcpapisv1alpha1.APIExport) *kcpapisv1alpha1.APIBinding { + t.Helper() + + apiBinding := &kcpapisv1alpha1.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: apiExport.Name, + }, + Spec: kcpapisv1alpha1.APIBindingSpec{ + Reference: kcpapisv1alpha1.BindingReference{ + Export: &kcpapisv1alpha1.ExportBindingReference{ + Path: string(logicalcluster.From(apiExport)), + Name: apiExport.Name, + }, + }, + // Specifying claims when the APIExport has none will lead to a condition + // on the APIBinding, but will not impact its functionality. + PermissionClaims: []kcpapisv1alpha1.AcceptablePermissionClaim{ + // the agent nearly always requires access to namespaces within workspaces + { + PermissionClaim: kcpapisv1alpha1.PermissionClaim{ + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "namespaces", + }, + All: true, + }, + State: kcpapisv1alpha1.ClaimAccepted, + }, + // for related resources, the agent can also sync ConfigMaps and Secrets + { + PermissionClaim: kcpapisv1alpha1.PermissionClaim{ + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "secrets", + }, + All: true, + }, + State: kcpapisv1alpha1.ClaimAccepted, + }, + { + PermissionClaim: kcpapisv1alpha1.PermissionClaim{ + GroupResource: kcpapisv1alpha1.GroupResource{ + Group: "", + Resource: "configmaps", + }, + All: true, + }, + State: kcpapisv1alpha1.ClaimAccepted, + }, + }, + }, + } + + t.Logf("Creating APIBinding %q…", apiBinding.Name) + if err := client.Create(ctx, apiBinding); err != nil { + t.Fatalf("Failed to create APIBinding: %v", err) + } + + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (done bool, err error) { + err = client.Get(ctx, ctrlruntimeclient.ObjectKeyFromObject(apiBinding), apiBinding) + if err != nil { + return false, err + } + + return conditions.IsTrue(apiBinding, conditionsv1alpha1.ReadyCondition), nil + }) + if err != nil { + t.Fatalf("Failed to wait for APIBinding virtual workspace to become ready: %v", err) + } + + return apiBinding +} diff --git a/test/utils/process.go b/test/utils/process.go new file mode 100644 index 0000000..d8be5fa --- /dev/null +++ b/test/utils/process.go @@ -0,0 +1,185 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "k8s.io/client-go/rest" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func requiredEnv(t *testing.T, name string) string { + t.Helper() + + value := os.Getenv(name) + if value == "" { + t.Fatalf("No $%s environment variable specified.", name) + } + + return value +} + +func ArtifactsDirectory(t *testing.T) string { + return requiredEnv(t, "ARTIFACTS") +} + +func AgentBinary(t *testing.T) string { + return requiredEnv(t, "AGENT_BINARY") +} + +var nonalpha = regexp.MustCompile(`[^a-z0-9_-]`) +var testCounters = map[string]int{} + +func uniqueLogfile(t *testing.T, basename string) string { + testName := strings.ToLower(t.Name()) + testName = nonalpha.ReplaceAllLiteralString(testName, "_") + testName = strings.Trim(testName, "_") + + if basename != "" { + testName += "_" + basename + } + + counter := testCounters[testName] + testCounters[testName]++ + + return fmt.Sprintf("%s_%02d.log", testName, counter) +} + +func RunAgent( + ctx context.Context, + t *testing.T, + name string, + kcpKubeconfig string, + localKubeconfig string, + apiExport string, +) context.CancelFunc { + t.Helper() + + t.Logf("Running agent %q…", name) + + args := []string{ + "--agent-name", name, + "--apiexport-ref", apiExport, + "--enable-leader-election=false", + "--kubeconfig", localKubeconfig, + "--kcp-kubeconfig", kcpKubeconfig, + "--namespace", "kube-system", + "--log-format", "Console", + } + + logFile := filepath.Join(ArtifactsDirectory(t), uniqueLogfile(t, "")) + log, err := os.Create(logFile) + if err != nil { + t.Fatalf("Failed to create logfile: %v", err) + } + + localCtx, cancel := context.WithCancel(ctx) + + cmd := exec.CommandContext(localCtx, AgentBinary(t), args...) + cmd.Stdout = log + cmd.Stderr = log + + if err := cmd.Start(); err != nil { + t.Fatalf("Failed to start api-syncagent: %v", err) + } + + cancelAndWait := func() { + cancel() + _ = cmd.Wait() + + log.Close() + } + + t.Cleanup(cancelAndWait) + + return cancelAndWait +} + +func RunEnvtest(t *testing.T, extraCRDs []string) (string, ctrlruntimeclient.Client, context.CancelFunc) { + t.Helper() + + testEnv := &envtest.Environment{ + ErrorIfCRDPathMissing: true, + } + + rootDirectory := requiredEnv(t, "ROOT_DIRECTORY") + extraCRDs = append(extraCRDs, "deploy/crd/kcp.io") + + for _, extra := range extraCRDs { + testEnv.CRDDirectoryPaths = append(testEnv.CRDDirectoryPaths, filepath.Join(rootDirectory, extra)) + } + + _, err := testEnv.Start() + if err != nil { + t.Fatalf("Failed to start envtest: %v", err) + } + + adminKubeconfig, adminRestConfig := createEnvtestKubeconfig(t, testEnv) + if err != nil { + t.Fatal(err) + } + + client, err := ctrlruntimeclient.New(adminRestConfig, ctrlruntimeclient.Options{ + Scheme: newScheme(t), + }) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + cancelAndWait := func() { + _ = testEnv.Stop() + } + + t.Cleanup(cancelAndWait) + + return adminKubeconfig, client, cancelAndWait +} + +func createEnvtestKubeconfig(t *testing.T, env *envtest.Environment) (string, *rest.Config) { + adminInfo := envtest.User{Name: "admin", Groups: []string{"system:masters"}} + + adminUser, err := env.ControlPlane.AddUser(adminInfo, nil) + if err != nil { + t.Fatal(err) + } + + adminKubeconfig, err := adminUser.KubeConfig() + if err != nil { + t.Fatal(err) + } + + kubeconfigFile, err := os.CreateTemp(os.TempDir(), "kubeconfig*") + if err != nil { + t.Fatalf("Failed to create envtest kubeconfig file: %v", err) + } + defer kubeconfigFile.Close() + + if _, err := kubeconfigFile.Write(adminKubeconfig); err != nil { + t.Fatalf("Failed to write envtest kubeconfig file: %v", err) + } + + return kubeconfigFile.Name(), adminUser.Config() +} diff --git a/test/utils/utils.go b/test/utils/utils.go new file mode 100644 index 0000000..054c89c --- /dev/null +++ b/test/utils/utils.go @@ -0,0 +1,174 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "net/url" + "os" + "regexp" + "testing" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + + kcpapisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + kcptenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/scale/scheme" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/kcp" +) + +func GetKcpAdminKubeconfig(t *testing.T) string { + return requiredEnv(t, "KCP_KUBECONFIG") +} + +func must(t *testing.T, err error) { + t.Helper() + + if err != nil { + t.Fatal(err) + } +} + +func newScheme(t *testing.T) *runtime.Scheme { + t.Helper() + + sc := runtime.NewScheme() + must(t, scheme.AddToScheme(sc)) + must(t, corev1.AddToScheme(sc)) + must(t, rbacv1.AddToScheme(sc)) + must(t, kcptenancyv1alpha1.AddToScheme(sc)) + must(t, kcpapisv1alpha1.AddToScheme(sc)) + must(t, syncagentv1alpha1.AddToScheme(sc)) + + return sc +} + +var clusterPathSuffix = regexp.MustCompile(`/clusters/[a-z0-9:*]+$`) + +func GetKcpAdminClusterClient(t *testing.T) ctrlruntimeclient.Client { + t.Helper() + return GetClusterClient(t, GetKcpAdminKubeconfig(t)) +} + +func GetClusterClient(t *testing.T, kubeconfig string) ctrlruntimeclient.Client { + t.Helper() + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + t.Fatalf("Failed to get load kubeconfig %q: %v", kubeconfig, err) + } + + // remove any pre-existing /clusters/... suffix, a cluster-aware client needs + // to point to the base URL (either of kcp or a virtual workspace) + config.Host = clusterPathSuffix.ReplaceAllLiteralString(config.Host, "") + + client, err := kcp.NewClusterAwareClient(config, ctrlruntimeclient.Options{ + Scheme: newScheme(t), + }) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + return client +} + +func GetKcpAdminClient(t *testing.T) ctrlruntimeclient.Client { + t.Helper() + return GetClient(t, GetKcpAdminKubeconfig(t)) +} + +func GetClient(t *testing.T, kubeconfig string) ctrlruntimeclient.Client { + t.Helper() + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + t.Fatalf("Failed to get load kubeconfig %q: %v", kubeconfig, err) + } + + client, err := ctrlruntimeclient.New(config, ctrlruntimeclient.Options{ + Scheme: newScheme(t), + }) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + return client +} + +func CreateKcpAgentKubeconfig(t *testing.T, path string) string { + t.Helper() + + agentToken := requiredEnv(t, "KCP_AGENT_TOKEN") + + kubeconfig, err := clientcmd.LoadFromFile(GetKcpAdminKubeconfig(t)) + if err != nil { + t.Fatalf("Failed to load admin kcp kubeconfig: %v", err) + } + + // drop everything but the currently selected context + if err := clientcmdapi.MinifyConfig(kubeconfig); err != nil { + t.Fatalf("Failed to minify admin kcp kubeconfig: %v", err) + } + + // update server URL if desired + if path != "" { + for name, cluster := range kubeconfig.Clusters { + parsed, err := url.Parse(cluster.Server) + if err != nil { + // Given how ultra lax url.Parse is, this basically never happens. + t.Fatalf("Failed to parse %q as URL: %v", cluster.Server, err) + } + + kubeconfig.Clusters[name].Server = fmt.Sprintf("%s://%s%s", parsed.Scheme, parsed.Host, path) + } + } + + // use the agent's token + for name := range kubeconfig.AuthInfos { + kubeconfig.AuthInfos[name].Token = agentToken + } + + // write the kubeconfig to a temporary file + encodedKubeconfig, err := clientcmd.Write(*kubeconfig) + if err != nil { + t.Fatalf("Failed to encode agent kubeconfig: %v", err) + } + + kubeconfigFile, err := os.CreateTemp(os.TempDir(), "kubeconfig*") + if err != nil { + t.Fatalf("Failed to create agent kubeconfig file: %v", err) + } + defer kubeconfigFile.Close() + + if _, err := kubeconfigFile.Write(encodedKubeconfig); err != nil { + t.Fatalf("Failed to write agent kubeconfig file: %v", err) + } + + // ensure the kubeconfig is removed after the test + t.Cleanup(func() { + os.Remove(kubeconfigFile.Name()) + }) + + return kubeconfigFile.Name() +} diff --git a/test/utils/wait.go b/test/utils/wait.go new file mode 100644 index 0000000..4094b8d --- /dev/null +++ b/test/utils/wait.go @@ -0,0 +1,42 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "testing" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func WaitForObject(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, obj ctrlruntimeclient.Object, key types.NamespacedName) { + t.Helper() + t.Logf("Waiting for %T to exist…", obj) + + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 3*time.Minute, false, func(ctx context.Context) (done bool, err error) { + err = client.Get(ctx, key, obj) + return err == nil, nil + }) + if err != nil { + t.Fatalf("Failed to wait for %T to exist: %v", obj, err) + } + + t.Logf("%T is ready.", obj) +}