Skip to content

Commit 7e334a3

Browse files
ryanroldsryanrolds
committed
K8s Gateway tests
1 parent d402691 commit 7e334a3

File tree

8 files changed

+235
-25
lines changed

8 files changed

+235
-25
lines changed

‎projects/gateway2/translator/plugins/httplisteneroptions/httplistener_options_plugin_test.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,17 @@ var _ = Describe("HttpListenerOptions Plugin", func() {
162162
})
163163
})
164164

165+
Describe("HttpListenerOptions Tracing", func() {
166+
When("OTEL tracing is enabled", func() {
167+
It("should not have authority set by default", func() {
168+
169+
})
170+
171+
It("should have authority set when specified", func() {
172+
173+
})
174+
})
175+
})
165176
})
166177

167178
func attachedHttpListenerOption() *solokubev1.HttpListenerOption {

‎projects/gloo/pkg/api/external/envoy/config/trace/v3/opentelemetry.pb.go‎

Lines changed: 4 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/kubernetes/e2e/features/tracing/suite.go‎

Lines changed: 121 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ tests by ensuring the console output is clean for each test.
5353
4. parse stdout from otelcol to see if the trace contains the data that we want
5454
*/
5555

56+
// SetupSuite installs the echo-server and curl pods
5657
func (s *testingSuite) SetupSuite() {
5758
var err error
5859

@@ -91,6 +92,7 @@ func (s *testingSuite) SetupSuite() {
9192
s.NoError(err, "can apply service/gateway-proxy-tracing")
9293
}
9394

95+
// TearDownSuite cleans up the resources created in SetupSuite
9496
func (s *testingSuite) TearDownSuite() {
9597
var err error
9698

@@ -105,6 +107,7 @@ func (s *testingSuite) TearDownSuite() {
105107
s.NoError(err, "can delete service/gateway-proxy-tracing")
106108
}
107109

110+
// BeforeTest sets up the common resources (otel, upstreams, virtual services)
108111
func (s *testingSuite) BeforeTest(string, string) {
109112
var err error
110113

@@ -138,10 +141,24 @@ func (s *testingSuite) BeforeTest(string, string) {
138141
core.Status_Accepted,
139142
gloo_defaults.GlooReporter,
140143
)
144+
}
145+
146+
// AfterTest cleans up the common resources (otel, upstreams, virtual services)
147+
func (s *testingSuite) AfterTest(string, string) {
148+
var err error
149+
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupOtelcolManifest)
150+
s.Assertions.NoError(err, "can delete otel collector")
141151

142-
err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gatewayConfigManifest,
152+
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, tracingConfigManifest)
153+
s.Assertions.NoError(err, "can delete gloo tracing config")
154+
}
155+
156+
// BeforeGlooGatewayTest sets up the Gloo Gateway resources
157+
func (s *testingSuite) BeforeGlooGatewayTest() {
158+
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gatewayConfigManifest,
143159
"-n", s.testInstallation.Metadata.InstallNamespace)
144160
s.NoError(err, "can create gateway and service")
161+
145162
s.testInstallation.Assertions.EventuallyResourceStatusMatchesState(
146163
func() (resources.InputResource, error) {
147164
return s.testInstallation.ResourceClients.GatewayClient().Read(
@@ -152,20 +169,39 @@ func (s *testingSuite) BeforeTest(string, string) {
152169
)
153170
}
154171

155-
func (s *testingSuite) AfterTest(string, string) {
156-
var err error
157-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupOtelcolManifest)
158-
s.Assertions.NoError(err, "can delete otel collector")
159-
160-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, tracingConfigManifest)
161-
s.Assertions.NoError(err, "can delete gloo tracing config")
162-
163-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayConfigManifest,
172+
// AfterGlooGatewayTest cleans up the Gloo Gateway resources
173+
func (s *testingSuite) AfterGlooGatewayTest() {
174+
err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayConfigManifest,
164175
"-n", s.testInstallation.Metadata.InstallNamespace)
165176
s.Assertions.NoError(err, "can delete gateway config")
166177
}
167178

168-
func (s *testingSuite) TestSpanNameTransformationsWithoutRouteDecorator() {
179+
// BeforeK8sGatewayTest sets up the K8s Gateway resources
180+
func (s *testingSuite) BeforeK8sGatewayTest(hloManifest string) {
181+
if !s.testInstallation.Metadata.K8sGatewayEnabled {
182+
s.T().Skip("Installation of Gloo Gateway does not have K8s Gateway enabled, skipping test as there is nothing to test")
183+
}
184+
185+
s.T().Cleanup(func() {
186+
err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, k8sGatewayManifest)
187+
s.Assertions.NoError(err, "cannot delete k8s gateway resources")
188+
189+
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, hloManifest)
190+
s.Assertions.NoError(err, "cannot delete k8s gateway hlo")
191+
})
192+
193+
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, hloManifest)
194+
s.Assertions.NoError(err, "can apply k8s gateway hlo")
195+
196+
err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, k8sGatewayManifest)
197+
s.Assertions.NoError(err, "can apply k8s gateway resources")
198+
199+
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment)
200+
}
201+
202+
func (s *testingSuite) TestGlooGatewaySpanNameTransformationsWithoutRouteDecorator() {
203+
s.BeforeGlooGatewayTest()
204+
169205
testHostname := "test-really-cool-hostname.com"
170206
s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
171207
[]curl.Option{
@@ -191,9 +227,13 @@ func (s *testingSuite) TestSpanNameTransformationsWithoutRouteDecorator() {
191227
// Name : <value of host header>
192228
assert.Regexp(c, "Name *: "+testHostname, logs)
193229
}, time.Second*30, time.Second*3, "otelcol logs contain span with name == hostname")
230+
231+
s.AfterGlooGatewayTest()
194232
}
195233

196-
func (s *testingSuite) TestSpanNameTransformationsWithRouteDecorator() {
234+
func (s *testingSuite) TestGlooGatewaySpanNameTransformationsWithRouteDecorator() {
235+
s.BeforeGlooGatewayTest()
236+
197237
s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
198238
[]curl.Option{
199239
curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{
@@ -218,9 +258,13 @@ func (s *testingSuite) TestSpanNameTransformationsWithRouteDecorator() {
218258
// Name : <value of routeDescriptorSpanName>
219259
assert.Regexp(c, "Name *: "+routeDescriptorSpanName, logs)
220260
}, time.Second*30, time.Second*3, "otelcol logs contain span with name == routeDescriptor")
261+
262+
s.AfterGlooGatewayTest()
221263
}
222264

223-
func (s *testingSuite) TestGatewayWithoutOtelTracingGrpcAuthority() {
265+
func (s *testingSuite) TestGlooGatewayWithoutOtelTracingGrpcAuthority() {
266+
s.BeforeGlooGatewayTest()
267+
224268
s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
225269
[]curl.Option{
226270
curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{
@@ -244,9 +288,20 @@ func (s *testingSuite) TestGatewayWithoutOtelTracingGrpcAuthority() {
244288
assert.Regexp(c, `-> authority: Str\(opentelemetry-collector_default\)`, logs)
245289
//s.Fail("this test is not implemented yet")
246290
}, time.Second*30, time.Second*3, "otelcol logs contain cluster name as authority")
291+
292+
s.AfterGlooGatewayTest()
247293
}
248294

249-
func (s *testingSuite) TestGatewayWithOtelTracingGrpcAuthority() {
295+
func (s *testingSuite) TestGlooGatewayWithOtelTracingGrpcAuthority() {
296+
s.BeforeGlooGatewayTest()
297+
298+
s.T().Cleanup(func() {
299+
// cleanup the gateway
300+
err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayAuthorityConfigManifest,
301+
"-n", s.testInstallation.Metadata.InstallNamespace)
302+
s.Assertions.NoError(err, "can delete gateway config")
303+
})
304+
250305
// create new gateway with grpc authority set
251306
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gatewayAuthorityConfigManifest,
252307
"-n", s.testInstallation.Metadata.InstallNamespace)
@@ -260,13 +315,6 @@ func (s *testingSuite) TestGatewayWithOtelTracingGrpcAuthority() {
260315
gloo_defaults.GlooReporter,
261316
)
262317

263-
s.T().Cleanup(func() {
264-
// cleanup the gateway
265-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayAuthorityConfigManifest,
266-
"-n", s.testInstallation.Metadata.InstallNamespace)
267-
s.Assertions.NoError(err, "can delete gateway config")
268-
})
269-
270318
s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(s.ctx, testdefaults.CurlPodExecOpt,
271319
[]curl.Option{
272320
curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{
@@ -292,3 +340,55 @@ func (s *testingSuite) TestGatewayWithOtelTracingGrpcAuthority() {
292340
//s.Fail("this test is not implemented yet")
293341
}, time.Second*30, time.Second*3, "otelcol logs contain authority set in gateway")
294342
}
343+
344+
func (s *testingSuite) TestK8sGatewayWithOtelTracing() {
345+
s.BeforeK8sGatewayTest(k8sGatewayHloTracingManifest)
346+
347+
s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(
348+
s.ctx,
349+
testdefaults.CurlPodExecOpt,
350+
[]curl.Option{
351+
curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
352+
curl.WithHostHeader("example.com"),
353+
curl.Silent(),
354+
},
355+
&matchers.HttpResponse{
356+
StatusCode: http.StatusOK,
357+
},
358+
5*time.Second, 30*time.Second,
359+
)
360+
361+
s.EventuallyWithT(func(c *assert.CollectT) {
362+
logs, err := s.testInstallation.Actions.Kubectl().GetContainerLogs(s.ctx,
363+
otelcolPod.ObjectMeta.GetNamespace(), otelcolPod.ObjectMeta.GetName())
364+
assert.NoError(c, err, "can get otelcol logs")
365+
366+
assert.Regexp(c, `-> authority: Str\(opentelemetry-collector_default\)`, logs)
367+
}, time.Second*30, time.Second*3, "otelcol logs contain cluster name as authority")
368+
}
369+
370+
func (s *testingSuite) TestK8sGatewayWithOtelTracingGrpcAuthority() {
371+
s.BeforeK8sGatewayTest(k8sGatewayHloAuthorityManifest)
372+
373+
s.testInstallation.Assertions.AssertEventuallyConsistentCurlResponse(
374+
s.ctx,
375+
testdefaults.CurlPodExecOpt,
376+
[]curl.Option{
377+
curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
378+
curl.WithHostHeader("example.com"),
379+
curl.Silent(),
380+
},
381+
&matchers.HttpResponse{
382+
StatusCode: http.StatusOK,
383+
},
384+
5*time.Second, 30*time.Second,
385+
)
386+
387+
s.EventuallyWithT(func(c *assert.CollectT) {
388+
logs, err := s.testInstallation.Actions.Kubectl().GetContainerLogs(s.ctx,
389+
otelcolPod.ObjectMeta.GetNamespace(), otelcolPod.ObjectMeta.GetName())
390+
assert.NoError(c, err, "can get otelcol logs")
391+
392+
assert.Regexp(c, `-> authority: Str\(test-authority\)`, logs)
393+
}, time.Second*30, time.Second*3, "otelcol logs contain authority set in gateway")
394+
}

‎test/kubernetes/e2e/features/tracing/testdata/k8s-gateway-hlo-tracing-authority.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
---
2+
apiVersion: gateway.solo.io/v1
3+
kind: HttpListenerOption
4+
metadata:
5+
name: gw-hlo-tracing-authority
6+
spec:
7+
targetRefs:
8+
- group: gateway.networking.k8s.io
9+
kind: Gateway
10+
name: gw
11+
namespace: default
12+
options:
13+
httpConnectionManagerSettings:
14+
tracing:
15+
openTelemetryConfig:
16+
collectorUpstreamRef:
17+
name: opentelemetry-collector
18+
namespace: default
19+
grpcService:
20+
authority: test-authority

‎test/kubernetes/e2e/features/tracing/testdata/k8s-gateway-hlo-tracing.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
apiVersion: gateway.solo.io/v1
3+
kind: HttpListenerOption
4+
metadata:
5+
name: gw-hlo-tracing-authority
6+
spec:
7+
targetRefs:
8+
- group: gateway.networking.k8s.io
9+
kind: Gateway
10+
name: gw
11+
namespace: default
12+
options:
13+
httpConnectionManagerSettings:
14+
tracing:
15+
openTelemetryConfig:
16+
collectorUpstreamRef:
17+
name: opentelemetry-collector
18+
namespace: default

‎test/kubernetes/e2e/features/tracing/testdata/k8s-gateway.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
kind: Gateway
3+
apiVersion: gateway.networking.k8s.io/v1
4+
metadata:
5+
name: gw
6+
spec:
7+
gatewayClassName: gloo-gateway
8+
listeners:
9+
- protocol: HTTP
10+
port: 8080
11+
name: http
12+
---
13+
apiVersion: gateway.networking.k8s.io/v1beta1
14+
kind: ReferenceGrant
15+
metadata:
16+
name: http-echo-grant
17+
namespace: http-echo
18+
spec:
19+
from:
20+
- group: gateway.networking.k8s.io
21+
kind: HTTPRoute
22+
namespace: default
23+
to:
24+
- group: ""
25+
kind: Service
26+
---
27+
apiVersion: gateway.networking.k8s.io/v1
28+
kind: HTTPRoute
29+
metadata:
30+
name: http-echo
31+
spec:
32+
parentRefs:
33+
- name: gw
34+
hostnames:
35+
- "example.com"
36+
rules:
37+
- backendRefs:
38+
- name: http-echo
39+
port: 3000
40+
namespace: http-echo

‎test/kubernetes/e2e/features/tracing/types.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"path/filepath"
55

66
"github.com/solo-io/skv2/codegen/util"
7+
appsv1 "k8s.io/api/apps/v1"
78
corev1 "k8s.io/api/core/v1"
89
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
910
)
@@ -16,14 +17,21 @@ const (
1617
gatewayProxyPort = 18080
1718
gatewayAuthorityProxyHost = "gateway-proxy-tracing-authority"
1819
gatewayAuthorityProxyPort = 18082
20+
k8sGatewayHost = "k8s-gateway-tracing"
21+
k8sGatewayPort = 8080
1922
)
2023

2124
var (
22-
setupOtelcolManifest = filepath.Join(util.MustGetThisDir(), "testdata", "setup-otelcol.yaml")
23-
tracingConfigManifest = filepath.Join(util.MustGetThisDir(), "testdata", "tracing.yaml")
25+
setupOtelcolManifest = filepath.Join(util.MustGetThisDir(), "testdata", "setup-otelcol.yaml")
26+
tracingConfigManifest = filepath.Join(util.MustGetThisDir(), "testdata", "tracing.yaml")
27+
gatewayProxyServiceManifest = filepath.Join(util.MustGetThisDir(), "testdata", "gw-proxy-tracing-service.yaml")
28+
2429
gatewayConfigManifest = filepath.Join(util.MustGetThisDir(), "testdata", "gateway.yaml")
2530
gatewayAuthorityConfigManifest = filepath.Join(util.MustGetThisDir(), "testdata", "gateway-authority.yaml")
26-
gatewayProxyServiceManifest = filepath.Join(util.MustGetThisDir(), "testdata", "gw-proxy-tracing-service.yaml")
31+
32+
k8sGatewayManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gateway.yaml")
33+
k8sGatewayHloTracingManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gateway-hlo-tracing.yaml")
34+
k8sGatewayHloAuthorityManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-gateway-hlo-tracing-authority.yaml")
2735

2836
otelcolPod = &corev1.Pod{
2937
ObjectMeta: metav1.ObjectMeta{Name: "otel-collector", Namespace: "default"},
@@ -39,4 +47,12 @@ var (
3947
Name: "virtual-service",
4048
Namespace: "default",
4149
}
50+
51+
// Proxy resource to be translated
52+
glooProxyObjectMeta = metav1.ObjectMeta{
53+
Name: "gloo-proxy-gw",
54+
Namespace: "default",
55+
}
56+
proxyDeployment = &appsv1.Deployment{ObjectMeta: glooProxyObjectMeta}
57+
proxyService = &corev1.Service{ObjectMeta: glooProxyObjectMeta}
4258
)

0 commit comments

Comments
 (0)