K8s Gateway tests

projects/gateway2/translator/plugins/httplisteneroptions/httplistener_options_plugin_test.go

@@ -162,6 +162,17 @@ var _ = Describe("HttpListenerOptions Plugin", func() {
 		})
 	})
 
+	Describe("HttpListenerOptions Tracing", func() {
+		When("OTEL tracing is enabled", func() {
+			It("should not have authority set by default", func() {
+
+			})
+
+			It("should have authority set when specified", func() {
+
+			})
+		})
+	})
 })
 
 func attachedHttpListenerOption() *solokubev1.HttpListenerOption {

test/kubernetes/e2e/features/tracing/suite.go

@@ -53,6 +53,7 @@ tests by ensuring the console output is clean for each test.
 4. parse stdout from otelcol to see if the trace contains the data that we want
 */
 
+// SetupSuite installs the echo-server and curl pods
 func (s *testingSuite) SetupSuite() {
 	var err error
 
@@ -91,6 +92,7 @@ func (s *testingSuite) SetupSuite() {
 	s.NoError(err, "can apply service/gateway-proxy-tracing")
 }
 
+// TearDownSuite cleans up the resources created in SetupSuite
 func (s *testingSuite) TearDownSuite() {
 	var err error
 
@@ -105,6 +107,7 @@ func (s *testingSuite) TearDownSuite() {
 	s.NoError(err, "can delete service/gateway-proxy-tracing")
 }
 
+// BeforeTest sets up the common resources (otel, upstreams, virtual services)
 func (s *testingSuite) BeforeTest(string, string) {
 	var err error
 
@@ -138,10 +141,24 @@ func (s *testingSuite) BeforeTest(string, string) {
 		core.Status_Accepted,
 		gloo_defaults.GlooReporter,
 	)
+}
+
+// AfterTest cleans up the common resources (otel, upstreams, virtual services)
+func (s *testingSuite) AfterTest(string, string) {
+	var err error
+	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupOtelcolManifest)
+	s.Assertions.NoError(err, "can delete otel collector")
 
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gatewayConfigManifest,
+	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, tracingConfigManifest)
+	s.Assertions.NoError(err, "can delete gloo tracing config")
+}
+
+// BeforeGlooGatewayTest sets up the Gloo Gateway resources
+func (s *testingSuite) BeforeGlooGatewayTest() {
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gatewayConfigManifest,
 		"-n", s.testInstallation.Metadata.InstallNamespace)
 	s.NoError(err, "can create gateway and service")
+
 	s.testInstallation.Assertions.EventuallyResourceStatusMatchesState(
 		func() (resources.InputResource, error) {
 			return s.testInstallation.ResourceClients.GatewayClient().Read(
@@ -152,20 +169,39 @@ func (s *testingSuite) BeforeTest(string, string) {
 	)
 }
 
-func (s *testingSuite) AfterTest(string, string) {
-	var err error
-	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupOtelcolManifest)
-	s.Assertions.NoError(err, "can delete otel collector")
-
-	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, tracingConfigManifest)
-	s.Assertions.NoError(err, "can delete gloo tracing config")
-
-	err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayConfigManifest,
+// AfterGlooGatewayTest cleans up the Gloo Gateway resources
+func (s *testingSuite) AfterGlooGatewayTest() {
+	err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayConfigManifest,
 		"-n", s.testInstallation.Metadata.InstallNamespace)
 	s.Assertions.NoError(err, "can delete gateway config")
 }
 
-func (s *testingSuite) TestSpanNameTransformationsWithoutRouteDecorator() {
+// BeforeK8sGatewayTest sets up the K8s Gateway resources
+func (s *testingSuite) BeforeK8sGatewayTest(hloManifest string) {
+	if !s.testInstallation.Metadata.K8sGatewayEnabled {
+		s.T().Skip("Installation of Gloo Gateway does not have K8s Gateway enabled, skipping test as there is nothing to test")
+	}
+
+	s.T().Cleanup(func() {
+		err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, k8sGatewayManifest)
+		s.Assertions.NoError(err, "cannot delete k8s gateway resources")
+
+		err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, hloManifest)
+		s.Assertions.NoError(err, "cannot delete k8s gateway hlo")
+	})
+
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, hloManifest)
+	s.Assertions.NoError(err, "can apply k8s gateway hlo")
+
+	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, k8sGatewayManifest)
+	s.Assertions.NoError(err, "can apply k8s gateway resources")
+
+	s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment)
+}
+
+func (s *testingSuite) TestGlooGatewaySpanNameTransformationsWithoutRouteDecorator() {
+	s.BeforeGlooGatewayTest()
+
 	testHostname := "test-really-cool-hostname.com"
 	s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
 		[]curl.Option{
@@ -191,9 +227,13 @@ func (s *testingSuite) TestSpanNameTransformationsWithoutRouteDecorator() {
 		// Name       : <value of host header>
 		assert.Regexp(c, "Name *: "+testHostname, logs)
 	}, time.Second*30, time.Second*3, "otelcol logs contain span with name == hostname")
+
+	s.AfterGlooGatewayTest()
 }
 
-func (s *testingSuite) TestSpanNameTransformationsWithRouteDecorator() {
+func (s *testingSuite) TestGlooGatewaySpanNameTransformationsWithRouteDecorator() {
+	s.BeforeGlooGatewayTest()
+
 	s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
 		[]curl.Option{
 			curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{
@@ -218,9 +258,13 @@ func (s *testingSuite) TestSpanNameTransformationsWithRouteDecorator() {
 		// Name       : <value of routeDescriptorSpanName>
 		assert.Regexp(c, "Name *: "+routeDescriptorSpanName, logs)
 	}, time.Second*30, time.Second*3, "otelcol logs contain span with name == routeDescriptor")
+
+	s.AfterGlooGatewayTest()
 }
 
-func (s *testingSuite) TestGatewayWithoutOtelTracingGrpcAuthority() {
+func (s *testingSuite) TestGlooGatewayWithoutOtelTracingGrpcAuthority() {
+	s.BeforeGlooGatewayTest()
+
 	s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
 		[]curl.Option{
 			curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{
@@ -244,9 +288,20 @@ func (s *testingSuite) TestGatewayWithoutOtelTracingGrpcAuthority() {
 		assert.Regexp(c, `-> authority: Str\(opentelemetry-collector_default\)`, logs)
 		//s.Fail("this test is not implemented yet")
 	}, time.Second*30, time.Second*3, "otelcol logs contain cluster name as authority")
+
+	s.AfterGlooGatewayTest()
 }
 
-func (s *testingSuite) TestGatewayWithOtelTracingGrpcAuthority() {
+func (s *testingSuite) TestGlooGatewayWithOtelTracingGrpcAuthority() {
+	s.BeforeGlooGatewayTest()
+
+	s.T().Cleanup(func() {
+		// cleanup the gateway
+		err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayAuthorityConfigManifest,
+			"-n", s.testInstallation.Metadata.InstallNamespace)
+		s.Assertions.NoError(err, "can delete gateway config")
+	})
+
 	// create new gateway with grpc authority set
 	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gatewayAuthorityConfigManifest,
 		"-n", s.testInstallation.Metadata.InstallNamespace)
@@ -260,13 +315,6 @@ func (s *testingSuite) TestGatewayWithOtelTracingGrpcAuthority() {
 		gloo_defaults.GlooReporter,
 	)
 
-	s.T().Cleanup(func() {
-		// cleanup the gateway
-		err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayAuthorityConfigManifest,
-			"-n", s.testInstallation.Metadata.InstallNamespace)
-		s.Assertions.NoError(err, "can delete gateway config")
-	})
-
 	s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
 		[]curl.Option{
 			curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{
@@ -292,3 +340,55 @@ func (s *testingSuite) TestGatewayWithOtelTracingGrpcAuthority() {
 		//s.Fail("this test is not implemented yet")
 	}, time.Second*30, time.Second*3, "otelcol logs contain authority set in gateway")
 }
+
+func (s *testingSuite) TestK8sGatewayWithOtelTracing() {
+	s.BeforeK8sGatewayTest(k8sGatewayHloTracingManifest)
+
+	s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(
+		s.ctx,
+		testdefaults.CurlPodExecOpt,
+		[]curl.Option{
+			curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
+			curl.WithHostHeader("example.com"),
+			curl.Silent(),
+		},
+		&matchers.HttpResponse{
+			StatusCode: http.StatusOK,
+		},
+		5*time.Second, 30*time.Second,
+	)
+
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		logs, err := s.testInstallation.Actions.Kubectl().GetContainerLogs(s.ctx,
+			otelcolPod.ObjectMeta.GetNamespace(), otelcolPod.ObjectMeta.GetName())
+		assert.NoError(c, err, "can get otelcol logs")
+
+		assert.Regexp(c, `-> authority: Str\(opentelemetry-collector_default\)`, logs)
+	}, time.Second*30, time.Second*3, "otelcol logs contain cluster name as authority")
+}
+
+func (s *testingSuite) TestK8sGatewayWithOtelTracingGrpcAuthority() {
+	s.BeforeK8sGatewayTest(k8sGatewayHloAuthorityManifest)
+
+	s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(
+		s.ctx,
+		testdefaults.CurlPodExecOpt,
+		[]curl.Option{
+			curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
+			curl.WithHostHeader("example.com"),
+			curl.Silent(),
+		},
+		&matchers.HttpResponse{
+			StatusCode: http.StatusOK,
+		},
+		5*time.Second, 30*time.Second,
+	)
+
+	s.EventuallyWithT(func(c *assert.CollectT) {
+		logs, err := s.testInstallation.Actions.Kubectl().GetContainerLogs(s.ctx,
+			otelcolPod.ObjectMeta.GetNamespace(), otelcolPod.ObjectMeta.GetName())
+		assert.NoError(c, err, "can get otelcol logs")
+
+		assert.Regexp(c, `-> authority: Str\(test-authority\)`, logs)
+	}, time.Second*30, time.Second*3, "otelcol logs contain authority set in gateway")
+}

test/kubernetes/e2e/features/tracing/testdata/k8s-gateway-hlo-tracing-authority.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: gateway.solo.io/v1
+kind: HttpListenerOption
+metadata:
+  name: gw-hlo-tracing-authority
+spec:
+  targetRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: gw
+    namespace: default
+  options:
+    httpConnectionManagerSettings:
+      tracing:
+        openTelemetryConfig:
+          collectorUpstreamRef:
+            name: opentelemetry-collector
+            namespace: default
+          grpcService:
+            authority: test-authority

test/kubernetes/e2e/features/tracing/testdata/k8s-gateway-hlo-tracing.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: gateway.solo.io/v1
+kind: HttpListenerOption
+metadata:
+  name: gw-hlo-tracing-authority
+spec:
+  targetRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: gw
+    namespace: default
+  options:
+    httpConnectionManagerSettings:
+      tracing:
+        openTelemetryConfig:
+          collectorUpstreamRef:
+            name: opentelemetry-collector
+            namespace: default

test/kubernetes/e2e/features/tracing/testdata/k8s-gateway.yaml

@@ -0,0 +1,40 @@
+---
+kind: Gateway
+apiVersion: gateway.networking.k8s.io/v1
+metadata:
+  name: gw
+spec:
+  gatewayClassName: gloo-gateway
+  listeners:
+    - protocol: HTTP
+      port: 8080
+      name: http
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: ReferenceGrant
+metadata:
+  name: http-echo-grant
+  namespace: http-echo
+spec:
+  from:
+  - group: gateway.networking.k8s.io
+    kind: HTTPRoute
+    namespace: default
+  to:
+  - group: ""
+    kind: Service
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-echo
+spec:
+  parentRefs:
+    - name: gw
+  hostnames:
+    - "example.com"
+  rules:
+    - backendRefs:
+        - name: http-echo
+          port: 3000
+          namespace: http-echo

test/kubernetes/e2e/features/tracing/types.go

@@ -4,6 +4,7 @@ import (
 	"path/filepath"
 
 	"github.com/solo-io/skv2/codegen/util"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -16,14 +17,21 @@ const (
 	gatewayProxyPort           = 18080
 	gatewayAuthorityProxyHost  = "gateway-proxy-tracing-authority"
 	gatewayAuthorityProxyPort  = 18082
+	k8sGatewayHost             = "k8s-gateway-tracing"
+	k8sGatewayPort             = 8080
 )
 
 var (
-	setupOtelcolManifest           = filepath.Join(util.MustGetThisDir(), "testdata", "setup-otelcol.yaml")
-	tracingConfigManifest          = filepath.Join(util.MustGetThisDir(), "testdata", "tracing.yaml")
+	setupOtelcolManifest        = filepath.Join(util.MustGetThisDir(), "testdata", "setup-otelcol.yaml")
+	tracingConfigManifest       = filepath.Join(util.MustGetThisDir(), "testdata", "tracing.yaml")
+	gatewayProxyServiceManifest = filepath.Join(util.MustGetThisDir(), "testdata", "gw-proxy-tracing-service.yaml")
+
 	gatewayConfigManifest          = filepath.Join(util.MustGetThisDir(), "testdata", "gateway.yaml")
 	gatewayAuthorityConfigManifest = filepath.Join(util.MustGetThisDir(), "testdata", "gateway-authority.yaml")
-	gatewayProxyServiceManifest    = filepath.Join(util.MustGetThisDir(), "testdata", "gw-proxy-tracing-service.yaml")
+
+	k8sGatewayManifest             = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gateway.yaml")
+	k8sGatewayHloTracingManifest   = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gateway-hlo-tracing.yaml")
+	k8sGatewayHloAuthorityManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gateway-hlo-tracing-authority.yaml")
 
 	otelcolPod = &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{Name: "otel-collector", Namespace: "default"},
@@ -39,4 +47,12 @@ var (
 		Name:      "virtual-service",
 		Namespace: "default",
 	}
+
+	// Proxy resource to be translated
+	glooProxyObjectMeta = metav1.ObjectMeta{
+		Name:      "gloo-proxy-gw",
+		Namespace: "default",
+	}
+	proxyDeployment = &appsv1.Deployment{ObjectMeta: glooProxyObjectMeta}
+	proxyService    = &corev1.Service{ObjectMeta: glooProxyObjectMeta}
 )