Skip to content

Commit 2f83b45

Browse files
committed
Run test containers on docker in docker on CircleCI
1 parent 78bfb6f commit 2f83b45

File tree

27 files changed

+357
-157
lines changed

27 files changed

+357
-157
lines changed

.circleci/autoforward.py

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
import dataclasses
4+
import threading
5+
import sys
6+
import signal
7+
import subprocess
8+
import json
9+
import re
10+
import time
11+
import logging
12+
13+
14+
@dataclasses.dataclass(frozen=True)
15+
class Forward:
16+
port: int
17+
18+
def __ne__(self, other):
19+
return not self.__eq__(other)
20+
21+
@staticmethod
22+
def parse_list(ports):
23+
r = []
24+
for port in ports.split(","):
25+
port_splits = port.split("->")
26+
if len(port_splits) < 2:
27+
continue
28+
host, ports = Forward.parse_host(port_splits[0], "localhost")
29+
for port in ports:
30+
r.append(Forward(port))
31+
return r
32+
33+
@staticmethod
34+
def parse_host(s, default_host):
35+
s = re.sub("/.*$", "", s)
36+
hp = s.split(":")
37+
if len(hp) == 1:
38+
return default_host, Forward.parse_ports(hp[0])
39+
if len(hp) == 2:
40+
return hp[0], Forward.parse_ports(hp[1])
41+
return None, []
42+
43+
@staticmethod
44+
def parse_ports(ports):
45+
port_range = ports.split("-")
46+
start = int(port_range[0])
47+
end = int(port_range[0]) + 1
48+
if len(port_range) > 2 or len(port_range) < 1:
49+
raise RuntimeError(f"don't know what to do with ports {ports}")
50+
if len(port_range) == 2:
51+
end = int(port_range[1]) + 1
52+
return list(range(start, end))
53+
54+
55+
class PortForwarder:
56+
def __init__(self, forward, local_bind_address="127.0.0.1"):
57+
self.process = subprocess.Popen(
58+
[
59+
"ssh",
60+
"-N",
61+
f"-L{local_bind_address}:{forward.port}:localhost:{forward.port}",
62+
"remote-docker",
63+
]
64+
)
65+
66+
def stop(self):
67+
self.process.kill()
68+
69+
70+
class DockerForwarder:
71+
def __init__(self):
72+
self.running = threading.Event()
73+
self.running.set()
74+
75+
def start(self):
76+
forwards = {}
77+
try:
78+
while self.running.is_set():
79+
new_forwards = self.container_config()
80+
existing_forwards = list(forwards.keys())
81+
for forward in new_forwards:
82+
if forward in existing_forwards:
83+
existing_forwards.remove(forward)
84+
else:
85+
logging.info(f"adding forward {forward}")
86+
forwards[forward] = PortForwarder(forward)
87+
88+
for to_clean in existing_forwards:
89+
logging.info(f"stopping forward {to_clean}")
90+
forwards[to_clean].stop()
91+
del forwards[to_clean]
92+
time.sleep(0.8)
93+
finally:
94+
for forward in forwards.values():
95+
forward.stop()
96+
97+
@staticmethod
98+
def container_config():
99+
def cmd(cmd_array):
100+
out = subprocess.Popen(
101+
cmd_array,
102+
universal_newlines=True,
103+
stdout=subprocess.PIPE,
104+
stderr=subprocess.PIPE,
105+
)
106+
out.wait()
107+
return out.communicate()[0]
108+
109+
try:
110+
stdout = cmd(["docker", "ps", "--format", "'{{json .}}'"])
111+
stdout = stdout.replace("'", "")
112+
configs = map(lambda l: json.loads(l), stdout.splitlines())
113+
forwards = []
114+
for c in configs:
115+
if c is None or c["Ports"] is None:
116+
continue
117+
ports = c["Ports"].strip()
118+
if ports == "":
119+
continue
120+
forwards += Forward.parse_list(ports)
121+
return forwards
122+
except RuntimeError:
123+
logging.error("Unexpected error:", sys.exc_info()[0])
124+
return []
125+
126+
def stop(self):
127+
logging.info("stopping")
128+
self.running.clear()
129+
130+
131+
def main():
132+
logging.basicConfig(
133+
format='%(asctime)s %(levelname)-8s %(message)s',
134+
level=logging.INFO,
135+
datefmt='%Y-%m-%d %H:%M:%S')
136+
forwarder = DockerForwarder()
137+
138+
def handler(*_):
139+
forwarder.stop()
140+
141+
signal.signal(signal.SIGINT, handler)
142+
143+
forwarder.start()
144+
145+
146+
if __name__ == "__main__":
147+
main()

.circleci/config.yml

+56-25
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,52 @@ commands:
8080
- attach_workspace:
8181
at: .
8282

83+
setup_testcontainers:
84+
description: >-
85+
Sets up remote docker and automatic port forwarding needed for docker on docker
86+
version of Testcontainers.
87+
steps:
88+
- setup_remote_docker:
89+
version: 20.10.14
90+
docker_layer_caching: true
91+
92+
- run:
93+
name: Show local ports
94+
command: |
95+
sudo sysctl net.ipv4.ip_local_port_range
96+
97+
- run:
98+
name: Show remote Docker ports
99+
command: |
100+
ssh remote-docker 'sudo sysctl net.ipv4.ip_local_port_range'
101+
102+
- run:
103+
name: Offset remote Docker ports to avoid tunnel collisions
104+
command: |
105+
ssh remote-docker 'sudo sysctl -w net.ipv4.ip_local_port_range="59095 60906"'
106+
107+
- run:
108+
name: Restart remote Docker
109+
command: |
110+
ssh remote-docker 'sudo sh -c "systemctl daemon-reload; systemctl restart docker"'
111+
112+
- run:
113+
name: Show remote Docker ports
114+
command: |
115+
ssh remote-docker 'sudo sysctl net.ipv4.ip_local_port_range'
116+
117+
- run:
118+
name: Testcontainers environment variables
119+
command: |
120+
echo "export TESTCONTAINERS_HOST_OVERRIDE=localhost" >> $BASH_ENV
121+
echo "export TESTCONTAINERS_RYUK_DISABLED=true" >> $BASH_ENV
122+
123+
- run:
124+
name: Testcontainers tunnels
125+
background: true
126+
command: |
127+
.circleci/autoforward.py
128+
83129
early_return_for_forked_pull_requests:
84130
description: >-
85131
If this build is from a fork, stop executing the current job and return success.
@@ -225,34 +271,21 @@ jobs:
225271

226272
docker:
227273
- image: *default_container
228-
# This is used by spymemcached instrumentation tests
229-
- image: memcached
230-
# This is used by rabbitmq instrumentation tests
231-
- image: rabbitmq
232-
# This is used by aerospike instrumentation tests
233-
- image: aerospike:5.5.0.9
234-
# This is used by mongodb instrumentation tests
235-
- image: mongo
236-
# This is used by jdbc and vert.x tests
237-
- image: mysql
238-
environment:
239-
MYSQL_ROOT_PASSWORD: password
240-
MYSQL_USER: sa
241-
MYSQL_PASSWORD: sa
242-
MYSQL_DATABASE: jdbcUnitTest
243-
# This is used by jdbc tests
244-
- image: postgres
245-
environment:
246-
POSTGRES_USER: sa
247-
POSTGRES_PASSWORD: sa
248-
POSTGRES_DB: jdbcUnitTest
249274

250275
steps:
251276
- setup_code
252277

253278
- restore_cache:
254279
<<: *cache_keys
255280

281+
- setup_testcontainers
282+
283+
- run:
284+
name: Start RabbitMQ (since the forked AmqpTests are having issues with Testcontainers)
285+
background: true
286+
command: |
287+
docker run -p 5672:5672 rabbitmq:3.9.20-alpine
288+
256289
- run:
257290
name: Run tests
258291
command: >-
@@ -295,17 +328,15 @@ jobs:
295328

296329
docker:
297330
- image: *default_container
298-
# This is used by rabbitmq smoke tests
299-
- image: rabbitmq
300-
# This is used by mongodb smoke tests
301-
- image: mongo
302331

303332
steps:
304333
- setup_code
305334

306335
- restore_cache:
307336
<<: *cache_keys
308337

338+
- setup_testcontainers
339+
309340
- run:
310341
name: Run Tests
311342
command: >-

dd-java-agent/instrumentation/aerospike-4/aerospike-4.gradle

+4
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,7 @@ dependencies {
2525

2626
latestDepTestImplementation group: 'com.aerospike', name: 'aerospike-client', version: '+'
2727
}
28+
29+
tasks.withType(Test).configureEach {
30+
usesService(testcontainersLimit)
31+
}

dd-java-agent/instrumentation/aerospike-4/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeAsyncClientTest.groovy

+3
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ import com.aerospike.client.async.EventLoops
88
import com.aerospike.client.async.NioEventLoops
99
import com.aerospike.client.listener.WriteListener
1010
import com.aerospike.client.policy.ClientPolicy
11+
import spock.lang.Requires
1112
import spock.lang.Shared
1213

1314
import static org.junit.Assert.fail
1415

16+
// Do not run tests on Java7 since testcontainers are not compatible with Java7
17+
@Requires({ jvm.java8Compatible })
1518
class AerospikeAsyncClientTest extends AerospikeBaseTest {
1619

1720
@Shared

dd-java-agent/instrumentation/aerospike-4/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy

+7-15
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,8 @@ import static datadog.trace.agent.test.utils.PortUtils.waitForPortToOpen
1313
import static java.util.concurrent.TimeUnit.SECONDS
1414
import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage
1515

16-
// Do not run tests locally on Java7 since testcontainers are not compatible with Java7
17-
// It is fine to run on CI because CI provides aerospike externally, not through testcontainers
18-
@Requires({ "true" == System.getenv("CI") || jvm.java8Compatible })
16+
// Do not run tests on Java7 since testcontainers are not compatible with Java7
17+
@Requires({ jvm.java8Compatible })
1918
abstract class AerospikeBaseTest extends AgentTestRunner {
2019

2120
@Shared
@@ -28,20 +27,13 @@ abstract class AerospikeBaseTest extends AgentTestRunner {
2827
int aerospikePort = 3000
2928

3029
def setup() throws Exception {
31-
/*
32-
CI will provide us with an aerospike container running alongside our build.
33-
When building locally, however, we need to take matters into our own hands
34-
and we use 'testcontainers' for this.
35-
*/
36-
if ("true" != System.getenv("CI")) {
37-
aerospike = new GenericContainer('aerospike:5.5.0.9')
38-
.withExposedPorts(3000)
39-
.waitingFor(forLogMessage(".*heartbeat-received.*\\n", 1))
30+
aerospike = new GenericContainer('aerospike:5.5.0.9')
31+
.withExposedPorts(3000)
32+
.waitingFor(forLogMessage(".*heartbeat-received.*\\n", 1))
4033

41-
aerospike.start()
34+
aerospike.start()
4235

43-
aerospikePort = aerospike.getMappedPort(3000)
44-
}
36+
aerospikePort = aerospike.getMappedPort(3000)
4537

4638
waitForPortToOpen(aerospikePort, 10, SECONDS)
4739
}

dd-java-agent/instrumentation/aerospike-4/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeClientTest.groovy

+3
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@ package datadog.trace.instrumentation.aerospike4
33
import com.aerospike.client.AerospikeClient
44
import com.aerospike.client.Bin
55
import com.aerospike.client.Key
6+
import spock.lang.Requires
67
import spock.lang.Shared
78

9+
// Do not run tests on Java7 since testcontainers are not compatible with Java7
10+
@Requires({ jvm.java8Compatible })
811
class AerospikeClientTest extends AerospikeBaseTest {
912

1013
@Shared

dd-java-agent/instrumentation/jdbc/jdbc.gradle

+4
Original file line numberDiff line numberDiff line change
@@ -80,3 +80,7 @@ tasks.named("test").configure {
8080
tasks.named("latestDepJava11Test").configure {
8181
javaLauncher = getJavaLauncherFor(11)
8282
}
83+
84+
tasks.withType(Test).configureEach {
85+
usesService(testcontainersLimit)
86+
}

dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy

+13-10
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import com.mchange.v2.c3p0.ComboPooledDataSource
22
import com.zaxxer.hikari.HikariConfig
33
import com.zaxxer.hikari.HikariDataSource
44
import datadog.trace.agent.test.AgentTestRunner
5+
import datadog.trace.agent.test.utils.PortUtils
56
import datadog.trace.api.DDSpanTypes
67
import datadog.trace.bootstrap.instrumentation.api.Tags
78
import org.testcontainers.containers.MySQLContainer
@@ -17,6 +18,7 @@ import java.sql.Driver
1718
import java.sql.PreparedStatement
1819
import java.sql.ResultSet
1920
import java.sql.Statement
21+
import java.util.concurrent.TimeUnit
2022

2123
import static datadog.trace.agent.test.utils.TraceUtils.basicSpan
2224
import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace
@@ -149,16 +151,17 @@ class RemoteJDBCInstrumentationTest extends AgentTestRunner {
149151
}
150152

151153
def setupSpec() {
152-
if (System.getenv("CI") != "true") {
153-
postgres = new PostgreSQLContainer("postgres:11.1")
154-
.withDatabaseName(dbName).withUsername("sa").withPassword("sa")
155-
postgres.start()
156-
jdbcUrls.put("postgresql", "${postgres.getJdbcUrl()}")
157-
mysql = new MySQLContainer("mysql:8.0")
158-
.withDatabaseName(dbName).withUsername("sa").withPassword("sa")
159-
mysql.start()
160-
jdbcUrls.put("mysql", "${mysql.getJdbcUrl()}")
161-
}
154+
postgres = new PostgreSQLContainer("postgres:11.1")
155+
.withDatabaseName(dbName).withUsername("sa").withPassword("sa")
156+
postgres.start()
157+
PortUtils.waitForPortToOpen(postgres.getHost(), postgres.getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT), 5, TimeUnit.SECONDS)
158+
jdbcUrls.put("postgresql", "${postgres.getJdbcUrl()}")
159+
mysql = new MySQLContainer("mysql:8.0")
160+
.withDatabaseName(dbName).withUsername("sa").withPassword("sa")
161+
mysql.start()
162+
PortUtils.waitForPortToOpen(mysql.getHost(), mysql.getMappedPort(MySQLContainer.MYSQL_PORT), 5, TimeUnit.SECONDS)
163+
jdbcUrls.put("mysql", "${mysql.getJdbcUrl()}")
164+
162165
prepareConnectionPoolDatasources()
163166
}
164167

0 commit comments

Comments
 (0)