Run test containers on docker in docker on CircleCI

Author: bantonsson

.circleci/autoforward.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import dataclasses
+import threading
+import sys
+import signal
+import subprocess
+import json
+import re
+import time
+import logging
+
+
+@dataclasses.dataclass(frozen=True)
+class Forward:
+  port: int
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+  @staticmethod
+  def parse_list(ports):
+    r = []
+    for port in ports.split(","):
+      port_splits = port.split("->")
+      if len(port_splits) < 2:
+        continue
+      host, ports = Forward.parse_host(port_splits[0], "localhost")
+      for port in ports:
+        r.append(Forward(port))
+    return r
+
+  @staticmethod
+  def parse_host(s, default_host):
+    s = re.sub("/.*$", "", s)
+    hp = s.split(":")
+    if len(hp) == 1:
+      return default_host, Forward.parse_ports(hp[0])
+    if len(hp) == 2:
+      return hp[0], Forward.parse_ports(hp[1])
+    return None, []
+
+  @staticmethod
+  def parse_ports(ports):
+    port_range = ports.split("-")
+    start = int(port_range[0])
+    end = int(port_range[0]) + 1
+    if len(port_range) > 2 or len(port_range) < 1:
+      raise RuntimeError(f"don't know what to do with ports {ports}")
+    if len(port_range) == 2:
+      end = int(port_range[1]) + 1
+    return list(range(start, end))
+
+
+class PortForwarder:
+  def __init__(self, forward, local_bind_address="127.0.0.1"):
+    self.process = subprocess.Popen(
+      [
+        "ssh",
+        "-N",
+        f"-L{local_bind_address}:{forward.port}:localhost:{forward.port}",
+        "remote-docker",
+      ]
+    )
+
+  def stop(self):
+    self.process.kill()
+
+
+class DockerForwarder:
+  def __init__(self):
+    self.running = threading.Event()
+    self.running.set()
+
+  def start(self):
+    forwards = {}
+    try:
+      while self.running.is_set():
+        new_forwards = self.container_config()
+        existing_forwards = list(forwards.keys())
+        for forward in new_forwards:
+          if forward in existing_forwards:
+            existing_forwards.remove(forward)
+          else:
+            logging.info(f"adding forward {forward}")
+            forwards[forward] = PortForwarder(forward)
+
+        for to_clean in existing_forwards:
+          logging.info(f"stopping forward {to_clean}")
+          forwards[to_clean].stop()
+          del forwards[to_clean]
+        time.sleep(0.8)
+    finally:
+      for forward in forwards.values():
+        forward.stop()
+
+  @staticmethod
+  def container_config():
+    def cmd(cmd_array):
+      out = subprocess.Popen(
+        cmd_array,
+        universal_newlines=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+      )
+      out.wait()
+      return out.communicate()[0]
+
+    try:
+      stdout = cmd(["docker", "ps", "--format", "'{{json .}}'"])
+      stdout = stdout.replace("'", "")
+      configs = map(lambda l: json.loads(l), stdout.splitlines())
+      forwards = []
+      for c in configs:
+        if c is None or c["Ports"] is None:
+          continue
+        ports = c["Ports"].strip()
+        if ports == "":
+          continue
+        forwards += Forward.parse_list(ports)
+      return forwards
+    except RuntimeError:
+      logging.error("Unexpected error:", sys.exc_info()[0])
+      return []
+
+  def stop(self):
+    logging.info("stopping")
+    self.running.clear()
+
+
+def main():
+  logging.basicConfig(
+    format='%(asctime)s %(levelname)-8s %(message)s',
+    level=logging.INFO,
+    datefmt='%Y-%m-%d %H:%M:%S')
+  forwarder = DockerForwarder()
+
+  def handler(*_):
+    forwarder.stop()
+
+  signal.signal(signal.SIGINT, handler)
+
+  forwarder.start()
+
+
+if __name__ == "__main__":
+  main()

.circleci/config.yml

@@ -80,6 +80,52 @@ commands:
       - attach_workspace:
           at: .
 
+  setup_testcontainers:
+    description: >-
+      Sets up remote docker and automatic port forwarding needed for docker on docker
+      version of Testcontainers.
+    steps:
+      - setup_remote_docker:
+          version: 20.10.14
+          docker_layer_caching: true
+
+      - run:
+          name: Show local ports
+          command: |
+            sudo sysctl net.ipv4.ip_local_port_range
+
+      - run:
+          name: Show remote Docker ports
+          command: |
+            ssh remote-docker 'sudo sysctl net.ipv4.ip_local_port_range'
+
+      - run:
+          name: Offset remote Docker ports to avoid tunnel collisions
+          command: |
+            ssh remote-docker 'sudo sysctl -w net.ipv4.ip_local_port_range="59095 60906"'
+
+      - run:
+          name: Restart remote Docker
+          command: |
+            ssh remote-docker 'sudo sh -c "systemctl daemon-reload; systemctl restart docker"'
+
+      - run:
+          name: Show remote Docker ports
+          command: |
+            ssh remote-docker 'sudo sysctl net.ipv4.ip_local_port_range'
+
+      - run:
+          name: Testcontainers environment variables
+          command: |
+            echo "export TESTCONTAINERS_HOST_OVERRIDE=localhost" >> $BASH_ENV
+            echo "export TESTCONTAINERS_RYUK_DISABLED=true" >> $BASH_ENV
+
+      - run:
+          name: Testcontainers tunnels
+          background: true
+          command: |
+            .circleci/autoforward.py
+
   early_return_for_forked_pull_requests:
     description: >-
       If this build is from a fork, stop executing the current job and return success.
@@ -225,34 +271,21 @@ jobs:
 
     docker:
       - image: *default_container
-        # This is used by spymemcached instrumentation tests
-      - image: memcached
-        # This is used by rabbitmq instrumentation tests
-      - image: rabbitmq
-        # This is used by aerospike instrumentation tests
-      - image: aerospike:5.5.0.9
-        # This is used by mongodb instrumentation tests
-      - image: mongo
-        # This is used by jdbc and vert.x tests
-      - image: mysql
-        environment:
-          MYSQL_ROOT_PASSWORD: password
-          MYSQL_USER: sa
-          MYSQL_PASSWORD: sa
-          MYSQL_DATABASE: jdbcUnitTest
-        # This is used by jdbc tests
-      - image: postgres
-        environment:
-          POSTGRES_USER: sa
-          POSTGRES_PASSWORD: sa
-          POSTGRES_DB: jdbcUnitTest
 
     steps:
       - setup_code
 
       - restore_cache:
           <<: *cache_keys
 
+      - setup_testcontainers
+
+      - run:
+          name: Start RabbitMQ (since the forked AmqpTests are having issues with Testcontainers)
+          background: true
+          command: |
+            docker run -p 5672:5672 rabbitmq:3.9.20-alpine
+
       - run:
           name: Run tests
           command: >-
@@ -295,17 +328,15 @@ jobs:
 
     docker:
       - image: *default_container
-        # This is used by rabbitmq smoke tests
-      - image: rabbitmq
-        # This is used by mongodb smoke tests
-      - image: mongo
 
     steps:
       - setup_code
 
       - restore_cache:
           <<: *cache_keys
 
+      - setup_testcontainers
+
       - run:
           name: Run Tests
           command: >-

dd-java-agent/instrumentation/aerospike-4/aerospike-4.gradle

@@ -25,3 +25,7 @@ dependencies {
 
   latestDepTestImplementation group: 'com.aerospike', name: 'aerospike-client', version: '+'
 }
+
+tasks.withType(Test).configureEach {
+  usesService(testcontainersLimit)
+}

dd-java-agent/instrumentation/aerospike-4/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeAsyncClientTest.groovy

@@ -8,10 +8,13 @@ import com.aerospike.client.async.EventLoops
 import com.aerospike.client.async.NioEventLoops
 import com.aerospike.client.listener.WriteListener
 import com.aerospike.client.policy.ClientPolicy
+import spock.lang.Requires
 import spock.lang.Shared
 
 import static org.junit.Assert.fail
 
+// Do not run tests on Java7 since testcontainers are not compatible with Java7
+@Requires({ jvm.java8Compatible })
 class AerospikeAsyncClientTest extends AerospikeBaseTest {
 
   @Shared

dd-java-agent/instrumentation/aerospike-4/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy

@@ -13,9 +13,8 @@ import static datadog.trace.agent.test.utils.PortUtils.waitForPortToOpen
 import static java.util.concurrent.TimeUnit.SECONDS
 import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage
 
-// Do not run tests locally on Java7 since testcontainers are not compatible with Java7
-// It is fine to run on CI because CI provides aerospike externally, not through testcontainers
-@Requires({ "true" == System.getenv("CI") || jvm.java8Compatible })
+// Do not run tests on Java7 since testcontainers are not compatible with Java7
+@Requires({ jvm.java8Compatible })
 abstract class AerospikeBaseTest extends AgentTestRunner {
 
   @Shared
@@ -28,20 +27,13 @@ abstract class AerospikeBaseTest extends AgentTestRunner {
   int aerospikePort = 3000
 
   def setup() throws Exception {
-    /*
-     CI will provide us with an aerospike container running alongside our build.
-     When building locally, however, we need to take matters into our own hands
-     and we use 'testcontainers' for this.
-     */
-    if ("true" != System.getenv("CI")) {
-      aerospike = new GenericContainer('aerospike:5.5.0.9')
-        .withExposedPorts(3000)
-        .waitingFor(forLogMessage(".*heartbeat-received.*\\n", 1))
+    aerospike = new GenericContainer('aerospike:5.5.0.9')
+      .withExposedPorts(3000)
+      .waitingFor(forLogMessage(".*heartbeat-received.*\\n", 1))
 
-      aerospike.start()
+    aerospike.start()
 
-      aerospikePort = aerospike.getMappedPort(3000)
-    }
+    aerospikePort = aerospike.getMappedPort(3000)
 
     waitForPortToOpen(aerospikePort, 10, SECONDS)
   }

dd-java-agent/instrumentation/aerospike-4/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeClientTest.groovy

@@ -3,8 +3,11 @@ package datadog.trace.instrumentation.aerospike4
 import com.aerospike.client.AerospikeClient
 import com.aerospike.client.Bin
 import com.aerospike.client.Key
+import spock.lang.Requires
 import spock.lang.Shared
 
+// Do not run tests on Java7 since testcontainers are not compatible with Java7
+@Requires({ jvm.java8Compatible })
 class AerospikeClientTest extends AerospikeBaseTest {
 
   @Shared

dd-java-agent/instrumentation/jdbc/jdbc.gradle

@@ -80,3 +80,7 @@ tasks.named("test").configure {
 tasks.named("latestDepJava11Test").configure {
   javaLauncher = getJavaLauncherFor(11)
 }
+
+tasks.withType(Test).configureEach {
+  usesService(testcontainersLimit)
+}

dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy

@@ -2,6 +2,7 @@ import com.mchange.v2.c3p0.ComboPooledDataSource
 import com.zaxxer.hikari.HikariConfig
 import com.zaxxer.hikari.HikariDataSource
 import datadog.trace.agent.test.AgentTestRunner
+import datadog.trace.agent.test.utils.PortUtils
 import datadog.trace.api.DDSpanTypes
 import datadog.trace.bootstrap.instrumentation.api.Tags
 import org.testcontainers.containers.MySQLContainer
@@ -17,6 +18,7 @@ import java.sql.Driver
 import java.sql.PreparedStatement
 import java.sql.ResultSet
 import java.sql.Statement
+import java.util.concurrent.TimeUnit
 
 import static datadog.trace.agent.test.utils.TraceUtils.basicSpan
 import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace
@@ -149,16 +151,17 @@ class RemoteJDBCInstrumentationTest extends AgentTestRunner {
   }
 
   def setupSpec() {
-    if (System.getenv("CI") != "true") {
-      postgres = new PostgreSQLContainer("postgres:11.1")
-        .withDatabaseName(dbName).withUsername("sa").withPassword("sa")
-      postgres.start()
-      jdbcUrls.put("postgresql", "${postgres.getJdbcUrl()}")
-      mysql = new MySQLContainer("mysql:8.0")
-        .withDatabaseName(dbName).withUsername("sa").withPassword("sa")
-      mysql.start()
-      jdbcUrls.put("mysql", "${mysql.getJdbcUrl()}")
-    }
+    postgres = new PostgreSQLContainer("postgres:11.1")
+      .withDatabaseName(dbName).withUsername("sa").withPassword("sa")
+    postgres.start()
+    PortUtils.waitForPortToOpen(postgres.getHost(), postgres.getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT), 5, TimeUnit.SECONDS)
+    jdbcUrls.put("postgresql", "${postgres.getJdbcUrl()}")
+    mysql = new MySQLContainer("mysql:8.0")
+      .withDatabaseName(dbName).withUsername("sa").withPassword("sa")
+    mysql.start()
+    PortUtils.waitForPortToOpen(mysql.getHost(), mysql.getMappedPort(MySQLContainer.MYSQL_PORT), 5, TimeUnit.SECONDS)
+    jdbcUrls.put("mysql", "${mysql.getJdbcUrl()}")
+
     prepareConnectionPoolDatasources()
   }