[feat] add distributed tracing using OTEL (#372)

* feat: added distributed tracing using OTEL Autoexport

Signed-off-by: Mateusz Urbanek <[email protected]>

* docs: added tracing chapter to docs

Signed-off-by: Mateusz Urbanek <[email protected]>

* feat: add gowrap to tools

Signed-off-by: Mateusz Urbanek <[email protected]>

* feat: add tracing to linodeclient

Signed-off-by: Mateusz Urbanek <[email protected]>

* fix: simplify the generated code signature

Signed-off-by: Mateusz Urbanek <[email protected]>

* fixup! fix: simplify the generated code signature

* fixup! feat: added distributed tracing using OTEL Autoexport

* test: add smoke tests to few parts of code

Signed-off-by: Mateusz Urbanek <[email protected]>

* chore: add codecov config file

Signed-off-by: Mateusz Urbanek <[email protected]>

---------

Signed-off-by: Mateusz Urbanek <[email protected]>

Author: shanduur

.golangci.yml

@@ -154,7 +154,6 @@ linters:
     # - goerr113
     - gofmt
     - goimports
-    - gomnd
     # - gocyclo
     - goprintffuncname
     - gosec
@@ -165,6 +164,7 @@ linters:
     - maintidx
     - makezero
     - misspell
+    - mnd
     - nestif
     - nilerr
     - nilnil

Dockerfile

@@ -18,6 +18,7 @@ COPY api/ api/
 COPY clients/ clients/
 COPY controller/ controller/
 COPY cloud/ cloud/
+COPY observability/ observability/
 COPY util/ util/
 COPY version/ version/
 
@@ -35,4 +36,7 @@ WORKDIR /
 COPY --from=builder /workspace/manager .
 USER 65532:65532
 
+# By default disable traces exporter
+ENV OTEL_TRACES_EXPORTER=none
+
 ENTRYPOINT ["/manager"]

Makefile

@@ -85,7 +85,8 @@ generate-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: generate-code
-generate-code: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate-code: controller-gen gowrap ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	go generate ./...
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 .PHONY: generate-mock
@@ -306,6 +307,7 @@ HUSKY          ?= $(LOCALBIN)/husky
 NILAWAY        ?= $(LOCALBIN)/nilaway
 GOVULNC        ?= $(LOCALBIN)/govulncheck
 MOCKGEN        ?= $(LOCALBIN)/mockgen
+GOWRAP         ?= $(CACHE_BIN)/gowrap
 
 ## Tool Versions
 KUSTOMIZE_VERSION        ?= v5.4.1
@@ -320,9 +322,10 @@ HUSKY_VERSION            ?= v0.2.16
 NILAWAY_VERSION          ?= latest
 GOVULNC_VERSION          ?= v1.1.1
 MOCKGEN_VERSION          ?= v0.4.0
+GOWRAP_VERSION           ?= latest
 
 .PHONY: tools
-tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(CONTROLLER_GEN) $(TILT) $(KIND) $(CHAINSAW) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC) $(MOCKGEN)
+tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(CONTROLLER_GEN) $(TILT) $(KIND) $(CHAINSAW) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC) $(MOCKGEN) $(GOWRAP)
 
 
 .PHONY: kustomize
@@ -375,7 +378,7 @@ $(CHAINSAW): $(CACHE_BIN)
 
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
-$(ENVTEST): $(LOCALBIN)
+$(ENVTEST): $(CACHE_BIN)
 	GOBIN=$(CACHE_BIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 
 .PHONY: husky
@@ -400,3 +403,8 @@ mockgen: $(MOCKGEN) ## Download mockgen locally if necessary.
 $(MOCKGEN): $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install go.uber.org/mock/mockgen@$(MOCKGEN_VERSION)
 
+.PHONY: gowrap
+gowrap: $(GOWRAP) ## Download gowrap locally if necessary.
+$(GOWRAP): $(CACHE_BIN)
+	GOBIN=$(CACHE_BIN) go install github.com/hexdigest/gowrap/cmd/gowrap@$(GOWRAP_VERSION)
+

Tiltfile

@@ -105,7 +105,7 @@ if debug == "true":
         "capl-compile",
         'GOOS=linux CGO_ENABLED=0 go build -gcflags "-N -l" -ldflags="-X github.com/linode/cluster-api-provider-linode/version.version=$VERSION" -a -o bin/manager ./cmd/main.go',
         deps=["./main.go", "./start.go", "vendor", "go.mod", "go.sum", "./api",  "./cloud", "./cmd", "./controller",
-            "./util", "./version",],
+            "./observability", "./util", "./version",],
         labels=["CAPL"],
     )
     docker_build_with_restart(
@@ -147,7 +147,7 @@ if os.getenv("SKIP_DOCKER_BUILD", "false") != "true" and debug != "true":
         "docker.io/linode/cluster-api-provider-linode",
         context=".",
         only=("Dockerfile", "Makefile", "vendor", "go.mod", "go.sum",
-        "./api", "./clients", "./cloud", "./cmd", "./controller", "./util", "./version",),
+        "./api", "./clients", "./cloud", "./cmd", "./controller", "./observability", "./util", "./version"),
         build_args={"VERSION": os.getenv("VERSION", "")},
     )
 

api/v1alpha1/linodecluster_webhook.go

@@ -74,6 +74,7 @@ func (r *LinodeCluster) ValidateDelete() (admission.Warnings, error) {
 }
 
 func (r *LinodeCluster) validateLinodeCluster(ctx context.Context, client LinodeClient) error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := r.validateLinodeClusterSpec(ctx, client); err != nil {
@@ -89,6 +90,7 @@ func (r *LinodeCluster) validateLinodeCluster(ctx context.Context, client Linode
 }
 
 func (r *LinodeCluster) validateLinodeClusterSpec(ctx context.Context, client LinodeClient) field.ErrorList {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := validateRegion(ctx, client, r.Spec.Region, field.NewPath("spec").Child("region")); err != nil {

api/v1alpha1/linodemachine_webhook.go

@@ -94,6 +94,7 @@ func (r *LinodeMachine) ValidateDelete() (admission.Warnings, error) {
 }
 
 func (r *LinodeMachine) validateLinodeMachine(ctx context.Context, client LinodeClient) error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := r.validateLinodeMachineSpec(ctx, client); err != nil {
@@ -109,6 +110,7 @@ func (r *LinodeMachine) validateLinodeMachine(ctx context.Context, client Linode
 }
 
 func (r *LinodeMachine) validateLinodeMachineSpec(ctx context.Context, client LinodeClient) field.ErrorList {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := validateRegion(ctx, client, r.Spec.Region, field.NewPath("spec").Child("region")); err != nil {

api/v1alpha1/linodeobjectstoragebucket_webhook.go

@@ -81,6 +81,7 @@ func (r *LinodeObjectStorageBucket) ValidateDelete() (admission.Warnings, error)
 }
 
 func (r *LinodeObjectStorageBucket) validateLinodeObjectStorageBucket(ctx context.Context, client LinodeClient) error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := r.validateLinodeObjectStorageBucketSpec(ctx, client); err != nil {
@@ -96,6 +97,7 @@ func (r *LinodeObjectStorageBucket) validateLinodeObjectStorageBucket(ctx contex
 }
 
 func (r *LinodeObjectStorageBucket) validateLinodeObjectStorageBucketSpec(ctx context.Context, client LinodeClient) field.ErrorList {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := validateObjectStorageCluster(ctx, client, r.Spec.Cluster, field.NewPath("spec").Child("cluster")); err != nil {

api/v1alpha1/linodevpc_webhook.go

@@ -113,6 +113,7 @@ func (r *LinodeVPC) ValidateDelete() (admission.Warnings, error) {
 }
 
 func (r *LinodeVPC) validateLinodeVPC(ctx context.Context, client LinodeClient) error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := r.validateLinodeVPCSpec(ctx, client); err != nil {
@@ -128,6 +129,7 @@ func (r *LinodeVPC) validateLinodeVPC(ctx context.Context, client LinodeClient)
 }
 
 func (r *LinodeVPC) validateLinodeVPCSpec(ctx context.Context, client LinodeClient) field.ErrorList {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := validateRegion(ctx, client, r.Spec.Region, field.NewPath("spec").Child("region"), LinodeVPCCapability); err != nil {

api/v1alpha1/webhook_helpers.go

@@ -11,6 +11,8 @@ import (
 	"github.com/linode/linodego"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
+	"github.com/linode/cluster-api-provider-linode/observability/wrappers/linodeclient"
+
 	. "github.com/linode/cluster-api-provider-linode/clients"
 )
 
@@ -21,12 +23,19 @@ const (
 	defaultClientTimeout = time.Second * 10
 )
 
+func mkptr[T any](v T) *T {
+	return &v
+}
+
 var (
 	// defaultLinodeClient is an unauthenticated Linode client
-	defaultLinodeClient = linodego.NewClient(&http.Client{Timeout: defaultClientTimeout})
+	defaultLinodeClient = linodeclient.NewLinodeClientWithTracing(
+		mkptr(linodego.NewClient(&http.Client{Timeout: defaultClientTimeout})),
+	)
 )
 
 func validateRegion(ctx context.Context, client LinodeClient, id string, path *field.Path, capabilities ...string) *field.Error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	region, err := client.GetRegion(ctx, id)
 	if err != nil {
 		return field.NotFound(path, id)
@@ -42,6 +51,7 @@ func validateRegion(ctx context.Context, client LinodeClient, id string, path *f
 }
 
 func validateLinodeType(ctx context.Context, client LinodeClient, id string, path *field.Path) (*linodego.LinodeType, *field.Error) {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	plan, err := client.GetType(ctx, id)
 	if err != nil {
 		return nil, field.NotFound(path, id)
@@ -61,6 +71,7 @@ func validateLinodeType(ctx context.Context, client LinodeClient, id string, pat
 // [Clusters List]: https://www.linode.com/docs/api/object-storage/#clusters-list
 // [Cluster View]: https://www.linode.com/docs/api/object-storage/#cluster-view
 func validateObjectStorageCluster(ctx context.Context, client LinodeClient, id string, path *field.Path) *field.Error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	//nolint:gocritic // prefer no escapes
 	cexp := regexp.MustCompile("^(([[:lower:]]+-)*[[:lower:]]+)-[[:digit:]]+$")
 	if !cexp.MatchString(id) {

api/v1alpha2/linodecluster_webhook.go

@@ -74,6 +74,7 @@ func (r *LinodeCluster) ValidateDelete() (admission.Warnings, error) {
 }
 
 func (r *LinodeCluster) validateLinodeCluster(ctx context.Context, client LinodeClient) error {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := r.validateLinodeClusterSpec(ctx, client); err != nil {
@@ -89,6 +90,7 @@ func (r *LinodeCluster) validateLinodeCluster(ctx context.Context, client Linode
 }
 
 func (r *LinodeCluster) validateLinodeClusterSpec(ctx context.Context, client LinodeClient) field.ErrorList {
+	// TODO: instrument with tracing, might need refactor to preserve readibility
 	var errs field.ErrorList
 
 	if err := validateRegion(ctx, client, r.Spec.Region, field.NewPath("spec").Child("region")); err != nil {

api/v1alpha2/webhook_helpers.go

@@ -26,6 +26,8 @@ import (
 	"github.com/linode/linodego"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
+	"github.com/linode/cluster-api-provider-linode/observability/wrappers/linodeclient"
+
 	. "github.com/linode/cluster-api-provider-linode/clients"
 )
 
@@ -36,9 +38,15 @@ const (
 	defaultClientTimeout = time.Second * 10
 )
 
+func mkptr[T any](v T) *T {
+	return &v
+}
+
 var (
 	// defaultLinodeClient is an unauthenticated Linode client
-	defaultLinodeClient = linodego.NewClient(&http.Client{Timeout: defaultClientTimeout})
+	defaultLinodeClient = linodeclient.NewLinodeClientWithTracing(
+		mkptr(linodego.NewClient(&http.Client{Timeout: defaultClientTimeout})),
+	)
 )
 
 func validateRegion(ctx context.Context, client LinodeClient, id string, path *field.Path, capabilities ...string) *field.Error {

cloud/scope/common.go

@@ -14,6 +14,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	"github.com/linode/cluster-api-provider-linode/observability/wrappers/linodeclient"
 	"github.com/linode/cluster-api-provider-linode/version"
 
 	. "github.com/linode/cluster-api-provider-linode/clients"
@@ -24,7 +25,19 @@ const (
 	defaultClientTimeout = time.Second * 10
 )
 
-func CreateLinodeClient(apiKey string, timeout time.Duration) (*linodego.Client, error) {
+type Option struct {
+	set func(client *linodego.Client)
+}
+
+func WithRetryCount(c int) Option {
+	return Option{
+		set: func(client *linodego.Client) {
+			client.SetRetryCount(c)
+		},
+	}
+}
+
+func CreateLinodeClient(apiKey string, timeout time.Duration, opts ...Option) (LinodeClient, error) {
 	if apiKey == "" {
 		return nil, errors.New("missing Linode API key")
 	}
@@ -41,7 +54,13 @@ func CreateLinodeClient(apiKey string, timeout time.Duration) (*linodego.Client,
 
 	linodeClient.SetUserAgent(fmt.Sprintf("CAPL/%s", version.GetVersion()))
 
-	return &linodeClient, nil
+	for _, opt := range opts {
+		opt.set(&linodeClient)
+	}
+
+	return linodeclient.NewLinodeClientWithTracing(
+		&linodeClient,
+	), nil
 }
 
 func getCredentialDataFromRef(ctx context.Context, crClient K8sClient, credentialsRef corev1.SecretReference, defaultNamespace string) ([]byte, error) {

cloud/scope/machine.go

@@ -84,11 +84,12 @@ func NewMachineScope(ctx context.Context, apiKey string, params MachineScopePara
 		}
 		apiKey = string(data)
 	}
-	linodeClient, err := CreateLinodeClient(apiKey, defaultClientTimeout)
+	linodeClient, err := CreateLinodeClient(apiKey, defaultClientTimeout,
+		WithRetryCount(0),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create linode client: %w", err)
 	}
-	linodeClient.SetRetryCount(0)
 
 	helper, err := patch.NewHelper(params.LinodeMachine, params.Client)
 	if err != nil {

cloud/scope/vpc.go

@@ -67,11 +67,12 @@ func NewVPCScope(ctx context.Context, apiKey string, params VPCScopeParams) (*VP
 		}
 		apiKey = string(data)
 	}
-	linodeClient, err := CreateLinodeClient(apiKey, defaultClientTimeout)
+	linodeClient, err := CreateLinodeClient(apiKey, defaultClientTimeout,
+		WithRetryCount(0),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create linode client: %w", err)
 	}
-	linodeClient.SetRetryCount(0)
 
 	helper, err := patch.NewHelper(params.LinodeVPC, params.Client)
 	if err != nil {