Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

e2e profiles #10634

Merged
merged 4 commits into from
Feb 17, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions test/kubernetes/e2e/example/manifests/example-profile.yaml
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fine leaving as-is for now but pretty much all values in here other than global are no longer relevant right?

Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# See kubernetes/e2e/tests/manifests/profiles for an explanation of test profiles

global:
image:
pullPolicy: IfNotPresent
Expand Down
14 changes: 0 additions & 14 deletions test/kubernetes/e2e/file.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build ignore

package e2e

import (
Expand All @@ -9,12 +7,6 @@ import (
)

var (
EdgeGatewayProfilePath = ProfilePath("edge-gateway.yaml")

KubernetesGatewayProfilePath = ProfilePath("kubernetes-gateway.yaml")

FullGatewayProfilePath = ProfilePath("full-gateway.yaml")

CommonRecommendationManifest = ManifestPath("common-recommendations.yaml")

// EmptyValuesManifestPath returns the path to a manifest with no values
Expand All @@ -33,9 +25,3 @@ func ManifestPath(pathParts ...string) string {
}, pathParts...)
return filepath.Join(manifestPathParts...)
}

// ProfilePath returns the absolute path to a profile file.
// These are all stored in the tests/manifests/profiles directory
func ProfilePath(path string) string {
return ManifestPath("profiles", path)
}
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/automtls_istio_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("istio-automtls-enabled-helm.yaml"),
},
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestTransformationValidationDisabled(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.EdgeGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("disable-transformation-validation-helm.yaml"),
},
)
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/discovery_watchlabels_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestDiscoveryWatchlabels(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: filepath.Join(fsutils.MustGetThisDir(), "manifests", "discovery-watchlabels-test-helm.yaml"),
},
)
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/full_envoy_validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestFullEnvoyValidation(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.FullGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("full-envoy-validation-helm.yaml"),
},
)
Expand Down
4 changes: 2 additions & 2 deletions test/kubernetes/e2e/tests/helm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestHelm(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.EdgeGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("test-helm.yaml"),
},
)
Expand Down Expand Up @@ -62,7 +62,7 @@ func TestHelmSettings(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.EdgeGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("test-helm.yaml"),
},
)
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/k8s_gw_aws_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestK8sGatewayAws(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("aws-lambda-helm.yaml"),
// these should correspond to the `settings.aws.*` values in the aws-lambda-helm.yaml manifest
AwsOptions: &kgateway.AwsOptions{
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/k8s_gw_istio_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestK8sGatewayIstio(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("istio-automtls-disabled-helm.yaml"),
},
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestK8sGatewayMinimalDefaultGatewayParameters(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml"),
},
)
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/k8s_gw_no_validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestK8sGatewayNoValidation(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("k8s-gateway-no-webhook-validation-test-helm.yaml"),
},
)
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/k8s_gw_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestK8sGateway(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.EmptyValuesManifestPath,
},
)
Expand Down
154 changes: 53 additions & 101 deletions test/kubernetes/e2e/tests/manifests/common-recommendations.yaml
Original file line number Diff line number Diff line change
@@ -1,117 +1,69 @@
# This file defines the common recommendations for a user who wants to run Gloo Gateway.
# As we call out in the [profiles README](./profiles/README.md), these should be reviewed and tested
# before being adopted.
# This file defines the common recommendations for a user who wants to run kgateway.
# These should be reviewed and tested before being adopted.
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this file is currently mostly commented out; need to figure out what the new helm values are. the current chart might not support all of them yet. filed #10635 to track

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in general think having a set of "production recommendation" is generally overkill for the current state of the project/helm chart, but keeping this file around mostly commented out makes sense to me in the short term

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 on profiles being a bit premature right now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah i don't know if there's much to add to this file for now, but eventually we might have some recommended values documented (like we did for edge in production) and probably want to test them here

#
# NOTE TO MAINTAINERS:
# Our Helm API has a set of default values. In most cases, we chose to not default _new_ APIs as a way
# of maintaining backwards compatibility. However, a side effect of that is that ideal configuration
# is not the default. That is where this file comes in.
# In this file we define any values which are not the default, but are the recommended pattern of installing,
# and thus testing, Gloo Gateway. Any values in this file should ideally be converted into
# and thus testing, kgateway. Any values in this file should ideally be converted into
# defaults in our values-template.yaml overtime. That way, users of the product can have the easiest experience,
# and gain the best value, without custom configuration.

# Configuration that is global to Gloo Gateway
global:
podSecurityStandards:
container:
enableRestrictedContainerDefaults: true

# Configuration for the Settings CR
settings:
# Gloo Gateway requires access to namespaces outside of the install namespace to watch and create Gateway resources
# singleNamespace=false must be set for namespace watch to work correctly. See: https://github.com/solo-io/solo-projects/issues/6058
# This is actually the default value, but we opted to include this because there is an issue with context around
# why this is the case
singleNamespace: false
invalidConfigPolicy:
# See https://docs.solo.io/gloo-edge/latest/guides/traffic_management/configuration_validation/invalid_route_replacement/ for more details
replaceInvalidRoutes: true

# Configuration for the Edge Gateway integration in Gloo Gateway
gateway:
# In earlier versions of Gloo Gateway (when it was named Gloo Edge) there were separate 'gloo' and 'gateway'
# pods, and the Proxy CR was persisted in etcD to allow that resource to be shared between the two pods.
# Those pods have since been combined, and persisting the resource in memory is unnecessary and costs
# unnecessary CPU to read/write and marshal/unmarshal
persistProxySpec: false
isolateVirtualHostsBySslConfig: true
validation:
# See https://github.com/solo-io/gloo/issues/9309 for more context around the default validation options,
# and some of the problems associated with it.
# Below, we define an appropriate set of defaults, so that our tests can rely on the preferred configuration
alwaysAcceptResources: false
warnRouteShortCircuiting: true
webhook:
# skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
skipDeleteValidationResources:
- upstreams

# Configuration for the Gloo pod
gloo:
# Configuration for the kgateway controller pod
controller:
# we need to be able to troubleshoot failures using logs; when users encounter problems enabling debug logging is
# a very early step in the troubleshooting process
logLevel: debug
deployment:
livenessProbeEnabled: true
customEnv:
# We scale the Gloo component frequently in tests, and need leadership to be established quickly
# Therefore, we set values to lower thresholds than the defaults
- name: LEADER_ELECTION_LEASE_DURATION
value: 4s
- name: LEADER_ELECTION_RENEW_PERIOD
value: 3s
- name: LEADER_ELECTION_RETRY_PERIOD
value: 1s
# Resource requests and limits should be identified by users directly
# We define them here to ensure that tests can pass with resources defined
resources:
requests:
# We have limited GitHub action resources which can cause containers to not create
# therefore we lessen the cpu resource requests values from the default (500m) to 100m.
cpu: 100m
memory: 256Mi
limits:
cpu: 1000m
memory: 10Gi
stats:
enabled: true # enable stats server for gloo so we can collect the metrics in CI
# TODO enable other values
# deployment:
# livenessProbeEnabled: true
# customEnv:
# # We scale the Gloo component frequently in tests, and need leadership to be established quickly
# # Therefore, we set values to lower thresholds than the defaults
# - name: LEADER_ELECTION_LEASE_DURATION
# value: 4s
# - name: LEADER_ELECTION_RENEW_PERIOD
# value: 3s
# - name: LEADER_ELECTION_RETRY_PERIOD
# value: 1s
# # Resource requests and limits should be identified by users directly
# # We define them here to ensure that tests can pass with resources defined
# resources:
# requests:
# # We have limited GitHub action resources which can cause containers to not create
# # therefore we lessen the cpu resource requests values from the default (500m) to 100m.
# cpu: 100m
# memory: 256Mi
# limits:
# cpu: 1000m
# memory: 10Gi
# stats:
# enabled: true # enable stats server for gloo so we can collect the metrics in CI

# Configuration for the statically deployed gateway-proxy that ships by default with Gloo Gateway
gatewayProxies:
gatewayProxy:
# TODO Configuration for the dynaimcally deployed proxies
# Panic mode allows Envoy load balancing to disregard host's health status.
# With panic mode enabled, it's possible that breaking changes to our health status configuration will still pass CI.
# As a result, we disable panic mode in our tests. This is not necessarily a recommendation for users running Gloo Gateway,
# As a result, we disable panic mode in our tests. This is not necessarily a recommendation for users running kgateway,
# but is a way of ensuring that our tests are validating health status configuration properly
# See https://github.com/solo-io/gloo/pull/3891 for more context
healthyPanicThreshold: 0
podTemplate:
# Enable a readiness probe
probes: true
# Enable a liveness probe
livenessProbeEnabled: true
# Graceful shutdown allows Envoy to begin failing healthchecks (that have been configured via the health check filter)
# and then sleep for a certain, customized amount of time.
# This is particularly when Gloo Gateway is running behind some external load-balancer, and we need to signal
# to that load balancer that the proxies should stop receiving traffic, without affecting the current traffic on the pods
# See https://github.com/solo-io/gloo/pull/3435 for more context
gracefulShutdown:
enabled: true
# This functionality isn't relied upon in our tests, but it's a good practice to define it,
# so we configure it here as an example
sleepTimeSeconds: 1
resources:
requests:
# We have limited GitHub action resources which can cause containers to not create
# therefore we lessen the cpu resource requests values from the default (500m) to 100m.
cpu: 100m
memory: 256Mi

# Configuration for the Discovery pod
discovery:
# Discovery is not recommended for production, so we disable it
# NOTE TO TEST AUTHORS: Upstreams must be statically defined in the manifests
enabled: false


# healthyPanicThreshold: 0
# podTemplate:
# # Enable a readiness probe
# probes: true
# # Enable a liveness probe
# livenessProbeEnabled: true
# # Graceful shutdown allows Envoy to begin failing healthchecks (that have been configured via the health check filter)
# # and then sleep for a certain, customized amount of time.
# # This is particularly when kgateway is running behind some external load-balancer, and we need to signal
# # to that load balancer that the proxies should stop receiving traffic, without affecting the current traffic on the pods
# gracefulShutdown:
# enabled: true
# # This functionality isn't relied upon in our tests, but it's a good practice to define it,
# # so we configure it here as an example
# sleepTimeSeconds: 1
# resources:
# requests:
# # We have limited GitHub action resources which can cause containers to not create
# # therefore we lessen the cpu resource requests values from the default (500m) to 100m.
# cpu: 100m
# memory: 256Mi
12 changes: 0 additions & 12 deletions test/kubernetes/e2e/tests/manifests/profiles/README.md

This file was deleted.

18 changes: 0 additions & 18 deletions test/kubernetes/e2e/tests/manifests/profiles/full-gateway.yaml

This file was deleted.

This file was deleted.

2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/revision_istio_k8s_gw_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestK8sGatewayIstioRevision(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("istio-revision-k8s-gateway-helm.yaml"),
},
)
Expand Down
4 changes: 2 additions & 2 deletions test/kubernetes/e2e/tests/upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestUpgradeFromLastPatchPreviousMinor(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: "upgrade-from-last-patch-previous-minor",
ProfileValuesManifestFile: e2e.FullGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.EmptyValuesManifestPath,
ReleasedVersion: lastPatchPreviousMinorVersion.String(),
},
Expand All @@ -54,7 +54,7 @@ func TestUpgradeFromCurrentPatchLatestMinor(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: "upgrade-from-current-patch-latest-minor",
ProfileValuesManifestFile: e2e.FullGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.EmptyValuesManifestPath,
ReleasedVersion: currentPatchMostRecentMinorVersion.String(),
},
Expand Down
2 changes: 1 addition & 1 deletion test/kubernetes/e2e/tests/watch_namespace_selector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func TestWatchNamespaceSelector(t *testing.T) {
t,
&kgateway.Context{
InstallNamespace: installNs,
ProfileValuesManifestFile: e2e.EdgeGatewayProfilePath,
ProfileValuesManifestFile: e2e.CommonRecommendationManifest,
ValuesManifestFile: e2e.ManifestPath("watch-namespace-selector.yaml"),
},
)
Expand Down
Loading
Loading