Skip to content

Commit 53fe59b

Browse files
randomandersonrobertomonteromiguel
randomanderson
and
robertomonteromiguel
authored
Configure Kind in Gitlab to allow kubernetes injection tests to run (#3057)
* implement logic to allow KinD to work in gitlab * try getting the container id directly * wrap with bash? * escape escape * Use json in python to get all the info * python doesn't handle bash piping too well * correct decoding of the json * more fixups * $HOME is not resolved * better the '/root' * extract to function * forgot a couple words * print out config * add debug info * add more debugging * fix typo * log running pods * test test-agent 1.16.0 * restore tag * pull if not present * restore pull policy * debug locally * deploy app * fix * fix local * debug local * no stop cluster * use internal dns to access to dev test agent * debug traces for gitlab patch * test * fix agent port * test manual inject * fix ports * fix * enable all tests * destroy cluster after * keep network * debug network connection * disable kind network policies * restore * diable tests * no pull images * load local image into cluster * no helm * revert helm charts * no destroy cluster * connect kind containers to bridget network * revert change * restore by default * test only helm * disable kube proxty * disable kube proxty * test * kubeproxy * pod subnet * connect kind cluster * pull offline * helm offline * cluster agent offline * preload webapp * pull policy never * enable all tests * run one by one * activate more tets * run one tests * test admission controller only * test uds * uds pull policy never * enable two tests * cluster agent traces * change interfaces sync * fix command sync * fix command sync * enable all tests * datadog kubernetes * fix merge * enable all * offline mode * helm chart offline mode file pattern * datadog helm offline * Remove offline-mode, rework setup * remove some unintended changes * some debug info. Fix sed * use formatting instead of a loop to get network info * strip() to remove whitespace * remove debug logs * merge and other fixes * formatting * text and variable name changes --------- Co-authored-by: roberto montero <[email protected]>
1 parent dd32a9b commit 53fe59b

File tree

3 files changed

+117
-51
lines changed

3 files changed

+117
-51
lines changed

‎tests/k8s_lib_injection/test_k8s_manual_inject.py

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -13,50 +13,52 @@
1313
class _TestAdmisionController:
1414
def test_inject_admission_controller(self, test_k8s_instance):
1515
logger.info(
16-
f"Launching test _test_inject_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
16+
f"Launching test _test_inject_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
1717
)
1818
test_k8s_instance.deploy_test_agent()
1919
test_k8s_instance.deploy_datadog_cluster_agent()
2020
test_k8s_instance.deploy_weblog_as_pod()
21-
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster.agent_port)
21+
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster)
2222
assert len(traces_json) > 0, "No traces found"
2323
logger.info(f"Test _test_inject_admission_controller finished")
2424

2525
def test_inject_uds_admission_controller(self, test_k8s_instance):
2626
logger.info(
27-
f"Launching test test_inject_uds_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
27+
f"Launching test test_inject_uds_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
2828
)
2929
test_k8s_instance.deploy_test_agent()
3030
test_k8s_instance.deploy_datadog_cluster_agent(use_uds=True)
3131
test_k8s_instance.deploy_weblog_as_pod()
32-
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster.agent_port)
32+
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster)
3333
assert len(traces_json) > 0, "No traces found"
3434
logger.info(f"Test test_inject_uds_admission_controller finished")
3535

3636
def test_inject_without_admission_controller(self, test_k8s_instance):
3737
logger.info(
38-
f"Launching test _test_inject_without_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
38+
f"Launching test _test_inject_without_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
3939
)
4040
test_k8s_instance.deploy_test_agent()
4141
test_k8s_instance.deploy_weblog_as_pod(with_admission_controller=False)
42-
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster.agent_port)
42+
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster)
4343
assert len(traces_json) > 0, "No traces found"
4444
logger.info(f"Test _test_inject_without_admission_controller finished")
4545

4646
def test_inject_uds_without_admission_controller(self, test_k8s_instance):
4747
logger.info(
48-
f"Launching test test_inject_uds_without_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
48+
f"Launching test test_inject_uds_without_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
4949
)
5050
test_k8s_instance.deploy_test_agent()
5151
test_k8s_instance.deploy_weblog_as_pod(with_admission_controller=False, use_uds=True)
52-
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster.agent_port)
52+
traces_json = self._get_dev_agent_traces(test_k8s_instance.k8s_kind_cluster)
5353
assert len(traces_json) > 0, "No traces found"
5454
logger.info(f"Test test_inject_uds_without_admission_controller finished")
5555

56-
def _get_dev_agent_traces(self, agent_port, retry=10):
56+
def _get_dev_agent_traces(self, k8s_kind_cluster, retry=10):
5757
for _ in range(retry):
5858
logger.info(f"[Check traces] Checking traces:")
59-
response = requests.get(f"http://localhost:{agent_port}/test/traces")
59+
response = requests.get(
60+
f"http://{k8s_kind_cluster.cluster_host_name}:{k8s_kind_cluster.get_agent_port()}/test/traces"
61+
)
6062
traces_json = response.json()
6163
if len(traces_json) > 0:
6264
logger.debug(f"Test traces response: {traces_json}")
@@ -73,7 +75,7 @@ class _TestAdmisionControllerAsm:
7375

7476
def test_inject_asm_admission_controller(self, test_k8s_instance):
7577
logger.info(
76-
f"Launching test test_inject_asm_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
78+
f"Launching test test_inject_asm_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
7779
)
7880

7981
asm_features = {
@@ -84,15 +86,16 @@ def test_inject_asm_admission_controller(self, test_k8s_instance):
8486
test_k8s_instance.deploy_datadog_cluster_agent(features=asm_features)
8587
test_k8s_instance.deploy_agent()
8688

87-
weblog_port = test_k8s_instance.k8s_kind_cluster.weblog_port
88-
logger.info(f"Waiting for weblog available [localhost:{weblog_port}]")
89-
wait_for_port(weblog_port, "localhost", 80.0)
90-
logger.info(f"[localhost:{weblog_port}]: Weblog app is ready!")
91-
warmup_weblog(f"http://localhost:{weblog_port}/")
92-
logger.info(f"Making a request to weblog [localhost:{weblog_port}]")
93-
request_uuid = make_get_request(f"http://localhost:{weblog_port}/")
89+
weblog_port = test_k8s_instance.k8s_kind_cluster.get_weblog_port()
90+
weblog_host = test_k8s_instance.k8s_kind_cluster.cluster_host_name
91+
logger.info(f"Waiting for weblog available [{weblog_host}:{weblog_port}]")
92+
wait_for_port(weblog_port, weblog_host, 80.0)
93+
logger.info(f"[{weblog_host}:{weblog_port}]: Weblog app is ready!")
94+
warmup_weblog(f"http://{weblog_host}:{weblog_port}/")
95+
logger.info(f"Making a request to weblog [{weblog_host}:{weblog_port}]")
96+
request_uuid = make_get_request(f"http://{weblog_host}:{weblog_port}/")
9497

95-
logger.info(f"Http request done with uuid: [{request_uuid}] for [localhost:{weblog_port}]")
98+
logger.info(f"Http request done with uuid: [{request_uuid}] for [{weblog_host}:{weblog_port}]")
9699
wait_backend_trace_id(request_uuid, 120.0, profile=False, validator=backend_trace_validator)
97100

98101

@@ -101,13 +104,15 @@ def test_inject_asm_admission_controller(self, test_k8s_instance):
101104
class TestAdmisionControllerProfiling:
102105
"""Test profiling activation with the admission controller."""
103106

104-
def _check_profiling_request_sent(self, agent_port, timeout=90):
107+
def _check_profiling_request_sent(self, k8s_kind_cluster, timeout=90):
105108
""" Use test agent profiling endpoint to check if the profiling data has been sent by the injectect library.
106109
Checks the request made to the profiling endpoint (/profiling/v1/input).
107110
The profiling post data can take between 12 and 90 seconds (12 if the library supports both env vars, 90 if it supports neither. """
108111
mustend = time.time() + timeout
109112
while time.time() < mustend:
110-
response = requests.get(f"http://localhost:{agent_port}/test/session/requests")
113+
response = requests.get(
114+
f"http://{k8s_kind_cluster.cluster_host_name}:{k8s_kind_cluster.get_agent_port()}/test/session/requests"
115+
)
111116
for request in response.json():
112117
if request["url"].endswith("/profiling/v1/input"):
113118
return True
@@ -117,36 +122,36 @@ def _check_profiling_request_sent(self, agent_port, timeout=90):
117122
def test_profiling_disabled_by_default(self, test_k8s_instance):
118123
logger.info(f"Launching test test_profiling_disabled_by_default")
119124
logger.info(
120-
f": Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
125+
f": Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
121126
)
122127
test_k8s_instance.deploy_test_agent()
123128
test_k8s_instance.deploy_datadog_cluster_agent()
124129
# if profiling is enabled force some profiling data to be sent
125130
test_k8s_instance.deploy_weblog_as_pod(
126131
env={"DD_PROFILING_UPLOAD_PERIOD": "10", "DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD": "1500"}
127132
)
128-
profiling_request_found = self._check_profiling_request_sent(test_k8s_instance.k8s_kind_cluster.agent_port)
133+
profiling_request_found = self._check_profiling_request_sent(test_k8s_instance.k8s_kind_cluster)
129134
assert not profiling_request_found, "Profiling should be disabled by default, but a profiling request was found"
130135

131136
@bug(context.library > "[email protected]", reason="APMON-1496")
132137
def test_profiling_admission_controller(self, test_k8s_instance):
133138
logger.info(f"Launching test test_profiling_admission_controller")
134139
logger.info(
135-
f": Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
140+
f": Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
136141
)
137142
test_k8s_instance.deploy_test_agent()
138143
test_k8s_instance.deploy_datadog_cluster_agent(features={"datadog.profiling.enabled": "auto"})
139144
test_k8s_instance.deploy_weblog_as_pod(
140145
env={"DD_PROFILING_UPLOAD_PERIOD": "10", "DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD": "1500"}
141146
)
142-
profiling_request_found = self._check_profiling_request_sent(test_k8s_instance.k8s_kind_cluster.agent_port)
147+
profiling_request_found = self._check_profiling_request_sent(test_k8s_instance.k8s_kind_cluster)
143148
assert profiling_request_found, "No profiling request found"
144149

145150
@bug(context.library > "[email protected]", reason="APMON-1496")
146151
def test_profiling_override_cluster_env(self, test_k8s_instance):
147152
logger.info(f"Launching test test_profiling_override_cluster_env")
148153
logger.info(
149-
f": Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
154+
f": Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
150155
)
151156
cluster_agent_config = {
152157
"clusterAgent.env[0].name": "DD_ADMISSION_CONTROLLER_AUTO_INSTRUMENTATION_PROFILING_ENABLED",
@@ -157,28 +162,29 @@ def test_profiling_override_cluster_env(self, test_k8s_instance):
157162
test_k8s_instance.deploy_weblog_as_pod(
158163
env={"DD_PROFILING_UPLOAD_PERIOD": "10", "DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD": "1500"}
159164
)
160-
profiling_request_found = self._check_profiling_request_sent(test_k8s_instance.k8s_kind_cluster.agent_port)
165+
profiling_request_found = self._check_profiling_request_sent(test_k8s_instance.k8s_kind_cluster)
161166
assert profiling_request_found, "No profiling request found"
162167

163168
def _test_inject_profiling_admission_controller_real(self, test_k8s_instance):
164169
logger.info(
165-
f"Launching test test_inject_profiling_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.weblog_port}] Agent: [{test_k8s_instance.k8s_kind_cluster.agent_port}]"
170+
f"Launching test test_inject_profiling_admission_controller: Weblog: [{test_k8s_instance.k8s_kind_cluster.get_weblog_port()}] Agent: [{test_k8s_instance.k8s_kind_cluster.get_agent_port()}]"
166171
)
167172

168173
test_k8s_instance.deploy_datadog_cluster_agent(features={"datadog.profiling.enabled": "auto"})
169174
test_k8s_instance.deploy_agent()
170175
test_k8s_instance.deploy_weblog_as_pod(
171176
env={"DD_PROFILING_UPLOAD_PERIOD": "10", "DD_INTERNAL_PROFILING_LONG_LIVED_THRESHOLD": "1500"}
172177
)
173-
weblog_port = test_k8s_instance.k8s_kind_cluster.weblog_port
174-
logger.info(f"Waiting for weblog available [localhost:{weblog_port}]")
175-
wait_for_port(weblog_port, "localhost", 80.0)
176-
logger.info(f"[localhost:{weblog_port}]: Weblog app is ready!")
177-
warmup_weblog(f"http://localhost:{weblog_port}/")
178-
logger.info(f"Making a request to weblog [localhost:{weblog_port}]")
179-
request_uuid = make_get_request(f"http://localhost:{weblog_port}/")
180-
181-
logger.info(f"Http request done with uuid: [{request_uuid}] for [localhost:{weblog_port}]")
178+
weblog_port = test_k8s_instance.k8s_kind_cluster.get_weblog_port()
179+
weblog_host = test_k8s_instance.k8s_kind_cluster.cluster_host_name
180+
logger.info(f"Waiting for weblog available [{weblog_host}:{weblog_port}]")
181+
wait_for_port(weblog_port, weblog_host, 80.0)
182+
logger.info(f"[{weblog_host}:{weblog_port}]: Weblog app is ready!")
183+
warmup_weblog(f"http://{weblog_host}:{weblog_port}/")
184+
logger.info(f"Making a request to weblog [{weblog_host}:{weblog_port}]")
185+
request_uuid = make_get_request(f"http://{weblog_host}:{weblog_port}/")
186+
187+
logger.info(f"Http request done with uuid: [{request_uuid}] for [{weblog_host}:{weblog_port}]")
182188
wait_backend_trace_id(request_uuid, 120.0, profile=True)
183189

184190

‎utils/k8s_lib_injection/k8s_command_utils.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
import subprocess, datetime, os, time, signal
1+
import subprocess, datetime, os, time, signal, shlex
22
from utils.tools import logger
33
from utils import context
44
from utils.k8s_lib_injection.k8s_sync_kubectl import KubectlLock
55
from retry import retry
66

77

8-
def execute_command(command, timeout=None, logfile=None):
8+
def execute_command(command, timeout=None, logfile=None, subprocess_env=None):
99
"""call shell-command and either return its output or kill it
1010
if it doesn't normally exit within timeout seconds and return None"""
1111
applied_timeout = 90
@@ -16,10 +16,16 @@ def execute_command(command, timeout=None, logfile=None):
1616
command_out_redirect = subprocess.PIPE
1717
if logfile:
1818
command_out_redirect = open(logfile, "w")
19+
20+
if not subprocess_env:
21+
subprocess_env = os.environ.copy()
22+
1923
output = ""
2024
try:
2125
start = datetime.datetime.now()
22-
process = subprocess.Popen(command.split(), stdout=command_out_redirect, stderr=command_out_redirect)
26+
process = subprocess.Popen(
27+
shlex.split(command), stdout=command_out_redirect, stderr=command_out_redirect, env=subprocess_env
28+
)
2329

2430
while process.poll() is None:
2531
time.sleep(0.1)

‎utils/k8s_lib_injection/k8s_kind_cluster.py

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import tempfile
66
from uuid import uuid4
77

8-
from utils.k8s_lib_injection.k8s_command_utils import execute_command
8+
from utils.k8s_lib_injection.k8s_command_utils import execute_command, execute_command_sync
99
from utils.tools import logger
1010
from utils import context
1111

@@ -21,7 +21,7 @@ def ensure_cluster():
2121

2222
def _ensure_cluster():
2323
k8s_kind_cluster = K8sKindCluster()
24-
k8s_kind_cluster.confiure_ports()
24+
k8s_kind_cluster.configure_networking(docker_in_docker="GITLAB_CI" in os.environ)
2525

2626
kind_data = ""
2727
with open("utils/k8s_lib_injection/resources/kind-config-template.yaml", "r") as file:
@@ -35,11 +35,18 @@ def _ensure_cluster():
3535
with open(cluster_config, "w") as fp:
3636
fp.write(kind_data)
3737
fp.seek(0)
38-
execute_command(
39-
f"kind create cluster --image=kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 --name {k8s_kind_cluster.cluster_name} --config {cluster_config} --wait 1m"
40-
)
4138

42-
# time.sleep(20)
39+
kind_command = f"kind create cluster --image=kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 --name {k8s_kind_cluster.cluster_name} --config {cluster_config} --wait 1m"
40+
41+
if "GITLAB_CI" in os.environ:
42+
# Kind needs to run in bridge network to communicate with the internet: https://github.com/DataDog/buildenv/blob/master/cookbooks/dd_firewall/templates/rules.erb#L96
43+
new_env = os.environ.copy()
44+
new_env["KIND_EXPERIMENTAL_DOCKER_NETWORK"] = "bridge"
45+
execute_command(kind_command, subprocess_env=new_env)
46+
47+
setup_kind_in_gitlab(k8s_kind_cluster)
48+
else:
49+
execute_command(kind_command)
4350

4451
return k8s_kind_cluster
4552

@@ -49,6 +56,37 @@ def destroy_cluster(k8s_kind_cluster):
4956
execute_command(f"docker rm -f {k8s_kind_cluster.cluster_name}-control-plane")
5057

5158

59+
def setup_kind_in_gitlab(k8s_kind_cluster):
60+
# The build runs in a docker container:
61+
# - Docker commands are forwarded to the host.
62+
# - The kind container is a sibling to the build container
63+
# Three things need to happen
64+
# 1) The kind container needs to be in the bridge network to communicate with the internet: done in _ensure_cluster()
65+
# 2) Kube config needs to be altered to use the correct IP of the control plane server
66+
# 3) The internal ports needs to be used rather than external ports: handled in get_agent_port() and get_weblog_port()
67+
correct_control_plane_ip = execute_command(
68+
f"docker container inspect {k8s_kind_cluster.cluster_name}-control-plane --format '{{{{.NetworkSettings.Networks.bridge.IPAddress}}}}'"
69+
).strip()
70+
if not correct_control_plane_ip:
71+
raise Exception("Unable to find correct control plane IP")
72+
logger.debug(f"[setup_kind_in_gitlab] correct_control_plane_ip: {correct_control_plane_ip}")
73+
74+
control_plane_address_in_config = execute_command(
75+
f'docker container inspect {k8s_kind_cluster.cluster_name}-control-plane --format \'{{{{index .NetworkSettings.Ports "6443/tcp" 0 "HostIp"}}}}:{{{{index .NetworkSettings.Ports "6443/tcp" 0 "HostPort"}}}}\''
76+
).strip()
77+
if not control_plane_address_in_config:
78+
raise Exception("Unable to find control plane address from config")
79+
logger.debug(f"[setup_kind_in_gitlab] control_plane_address_in_config: {control_plane_address_in_config}")
80+
81+
# Replace server config with dns name + internal port
82+
execute_command_sync(
83+
f"sed -i -e 's/{control_plane_address_in_config}/{correct_control_plane_ip}:6443/g' {os.environ['HOME']}/.kube/config",
84+
k8s_kind_cluster,
85+
)
86+
87+
k8s_kind_cluster.cluster_host_name = correct_control_plane_ip
88+
89+
5290
def get_free_port():
5391
last_allowed_port = 65535
5492
port = random.randint(1100, 65100)
@@ -67,10 +105,26 @@ class K8sKindCluster:
67105
def __init__(self):
68106
self.cluster_name = f"lib-injection-testing-{str(uuid4())[:8]}"
69107
self.context_name = f"kind-{self.cluster_name}"
70-
self.agent_port = 18126
71-
self.weblog_port = 18080
72-
73-
def confiure_ports(self):
74-
# Get random free ports
108+
self.cluster_host_name = "localhost"
109+
self.agent_port = None
110+
self.weblog_port = None
111+
self.internal_agent_port = None
112+
self.internal_weblog_port = None
113+
self.docker_in_docker = False
114+
115+
def configure_networking(self, docker_in_docker=False):
116+
self.docker_in_docker = docker_in_docker
75117
self.agent_port = get_free_port()
76118
self.weblog_port = get_free_port()
119+
self.internal_agent_port = 8126
120+
self.internal_weblog_port = 18080
121+
122+
def get_agent_port(self):
123+
if self.docker_in_docker:
124+
return self.internal_agent_port
125+
return self.agent_port
126+
127+
def get_weblog_port(self):
128+
if self.docker_in_docker:
129+
return self.internal_weblog_port
130+
return self.weblog_port

0 commit comments

Comments
 (0)