From 32da732337dedc5ad98d2206f5d9779ce08c9fb1 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 4 Feb 2025 02:39:36 -0500 Subject: [PATCH 1/6] Add a configured password --- .../com/clickhouse/kafka/connect/sink/ClickHouseBase.java | 2 +- .../connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java | 5 ++++- .../connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java index 88f0a32d..71e0daa3 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java @@ -34,7 +34,7 @@ public static void setup() throws IOException { initialPing(); return; } - db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_DOCKER_IMAGE); + db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_DOCKER_IMAGE).withEnv("CLICKHOUSE_PASSWORD", "test_password"); db.start(); } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java index 07bb0433..db021b84 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java @@ -29,7 +29,10 @@ public class ClickHouseSinkTaskSchemalessProxyTest extends ClickHouseBase { public static void setup() throws IOException { Network network = Network.newNetwork(); // Note: we are using a different version of ClickHouse for the proxy - https://github.com/ClickHouse/ClickHouse/issues/58828 - db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE).withNetwork(network).withNetworkAliases("clickhouse"); + db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE) + .withNetwork(network) + .withNetworkAliases("clickhouse") + .withEnv("CLICKHOUSE_PASSWORD", "test_password"); db.start(); toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.7.0").withNetwork(network).withNetworkAliases("toxiproxy"); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java index 72b79499..b5da1e2b 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java @@ -39,7 +39,10 @@ public class ClickHouseSinkTaskWithSchemaProxyTest extends ClickHouseBase { public static void setup() throws IOException { Network network = Network.newNetwork(); // Note: we are using a different version of ClickHouse for the proxy - https://github.com/ClickHouse/ClickHouse/issues/58828 - db = new org.testcontainers.clickhouse.ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE).withNetwork(network).withNetworkAliases("clickhouse"); + db = new org.testcontainers.clickhouse.ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE) + .withNetwork(network) + .withNetworkAliases("clickhouse") + .withEnv("CLICKHOUSE_PASSWORD", "test_password"); db.start(); toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.7.0").withNetwork(network).withNetworkAliases("toxiproxy"); From 4d987fb2f791be9c7afa487cac40cd0ac8beba98 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 4 Feb 2025 02:57:10 -0500 Subject: [PATCH 2/6] Setting password differently --- .../java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java | 2 +- .../connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java | 2 +- .../connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java index 71e0daa3..bd398c73 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java @@ -34,7 +34,7 @@ public static void setup() throws IOException { initialPing(); return; } - db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_DOCKER_IMAGE).withEnv("CLICKHOUSE_PASSWORD", "test_password"); + db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_DOCKER_IMAGE).withPassword("test_password"); db.start(); } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java index db021b84..4e0b9e9e 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java @@ -32,7 +32,7 @@ public static void setup() throws IOException { db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE) .withNetwork(network) .withNetworkAliases("clickhouse") - .withEnv("CLICKHOUSE_PASSWORD", "test_password"); + .withPassword("test_password"); db.start(); toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.7.0").withNetwork(network).withNetworkAliases("toxiproxy"); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java index b5da1e2b..3308d740 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java @@ -42,7 +42,7 @@ public static void setup() throws IOException { db = new org.testcontainers.clickhouse.ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE) .withNetwork(network) .withNetworkAliases("clickhouse") - .withEnv("CLICKHOUSE_PASSWORD", "test_password"); + .withPassword("test_password"); db.start(); toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.7.0").withNetwork(network).withNetworkAliases("toxiproxy"); From 0e1c47bf5eea7b8acb1c3a20c8f0534ef3890d38 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 4 Feb 2025 03:07:30 -0500 Subject: [PATCH 3/6] Adding a new environment variable --- .../java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java | 2 +- .../connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java | 3 ++- .../connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java index bd398c73..2931fccd 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseBase.java @@ -34,7 +34,7 @@ public static void setup() throws IOException { initialPing(); return; } - db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_DOCKER_IMAGE).withPassword("test_password"); + db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_DOCKER_IMAGE).withPassword("test_password").withEnv("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1"); db.start(); } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java index 4e0b9e9e..24a46350 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskSchemalessProxyTest.java @@ -32,7 +32,8 @@ public static void setup() throws IOException { db = new ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE) .withNetwork(network) .withNetworkAliases("clickhouse") - .withPassword("test_password"); + .withPassword("test_password") + .withEnv("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1"); db.start(); toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.7.0").withNetwork(network).withNetworkAliases("toxiproxy"); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java index 3308d740..f6f5b1cc 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaProxyTest.java @@ -42,7 +42,8 @@ public static void setup() throws IOException { db = new org.testcontainers.clickhouse.ClickHouseContainer(ClickHouseTestHelpers.CLICKHOUSE_FOR_PROXY_DOCKER_IMAGE) .withNetwork(network) .withNetworkAliases("clickhouse") - .withPassword("test_password"); + .withPassword("test_password") + .withEnv("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1"); db.start(); toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.7.0").withNetwork(network).withNetworkAliases("toxiproxy"); From 7532f2d2ab7aef0f6f70fc263ddfb753d130cc50 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 4 Feb 2025 03:32:50 -0500 Subject: [PATCH 4/6] Update ClickHouseTestHelpers.java --- .../kafka/connect/sink/helper/ClickHouseTestHelpers.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java b/src/test/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java index 23eba246..6698c806 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java @@ -183,7 +183,7 @@ public static OperationMetrics optimizeTable(ClickHouseHelperClient chc, String } public static int countRows(ClickHouseHelperClient chc, String tableName) { - String queryCount = String.format("SELECT COUNT(*) FROM `%s`", tableName); + String queryCount = String.format("SELECT COUNT(*) FROM `%s` SETTINGS select_sequential_consistency = 1", tableName); try { optimizeTable(chc, tableName); @@ -212,7 +212,7 @@ public static int sumRows(ClickHouseHelperClient chc, String tableName, String c } public static int countRowsWithEmojis(ClickHouseHelperClient chc, String tableName) { - String queryCount = "SELECT COUNT(*) FROM `" + tableName + "` WHERE str LIKE '%\uD83D\uDE00%'"; + String queryCount = "SELECT COUNT(*) FROM `" + tableName + "` WHERE str LIKE '%\uD83D\uDE00%' SETTINGS select_sequential_consistency = 1"; try { Records records = chc.getClient().queryRecords(queryCount).get(CLOUD_TIMEOUT_VALUE, CLOUD_TIMEOUT_UNIT); String value = records.iterator().next().getString(1); From fbcd881b838557c8d6dfd5b2a352a268c0ebccfb Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 4 Feb 2025 03:39:47 -0500 Subject: [PATCH 5/6] Update ClickHouseSinkTaskStringTest.java --- .../kafka/connect/sink/ClickHouseSinkTaskStringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java index 057e56d6..dc91ebae 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java @@ -35,7 +35,7 @@ private int countRowsWithEmojis(ClickHouseHelperClient chc, String topic) { } } private int countRows(ClickHouseHelperClient chc, String topic) { - String queryCount = String.format("select count(*) from `%s`", topic); + String queryCount = String.format("select count(*) from `%s` SETTINGS select_sequential_consistency = 1", topic); try { Records records = chc.getClient().queryRecords(queryCount).get(); String value = records.iterator().next().getString(1); From d29102c7bb04f65594905afc2e8f2c5ae7d66f1b Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 4 Feb 2025 03:46:43 -0500 Subject: [PATCH 6/6] Update ClickHouseSinkTaskStringTest.java --- .../kafka/connect/sink/ClickHouseSinkTaskStringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java index dc91ebae..c1b3d499 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskStringTest.java @@ -23,7 +23,7 @@ public class ClickHouseSinkTaskStringTest extends ClickHouseBase { private int countRowsWithEmojis(ClickHouseHelperClient chc, String topic) { - String queryCount = "select count(*) from " + topic + " where str LIKE '%\uD83D\uDE00%'"; + String queryCount = "select count(*) from " + topic + " where str LIKE '%\uD83D\uDE00%' SETTINGS select_sequential_consistency = 1"; try { Records records = chc.getClient().queryRecords(queryCount).get(); String value = records.iterator().next().getString(1);