Skip to content

Commit 08058fe

Browse files
zhicwuzhicwu
authored
Merge pull request #783 from zhicwu/enhance-jdbc
Prepare 0.3.2
2 parents 48a6281 + 82868de commit 08058fe

File tree

67 files changed

+3030
-922
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+3030
-922
lines changed

‎.github/workflows/benchmark.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
mvn --batch-mode --update-snapshots -q -DskipTests install
6363
cd clickhouse-benchmark
6464
java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf text \
65-
-p client=clickhouse-http-jdbc -p client=clickhouse-grpc-jdbc -p type=default Basic
65+
-p client=clickhouse-http-jdbc1 -p client=clickhouse-grpc-jdbc -p type=object Basic
6666
echo "BENCHMARK_REPORT<<EOF" >> $GITHUB_ENV
6767
cat jmh-result.text >> $GITHUB_ENV
6868
echo "EOF" >> $GITHUB_ENV

‎README.md

Lines changed: 150 additions & 151 deletions
Large diffs are not rendered by default.

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public void doSetup(ServerState serverState) throws Exception {
8888
client = createClient();
8989

9090
String[] sqls = new String[] { "drop table if exists system.test_insert",
91-
"create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory" };
91+
"create table if not exists system.test_insert(id String, i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory" };
9292

9393
for (String sql : sqls) {
9494
try (ClickHouseResponse resp = client.connect(server).query(sql).execute().get()) {
@@ -98,10 +98,13 @@ public void doSetup(ServerState serverState) throws Exception {
9898
}
9999

100100
@TearDown(Level.Trial)
101-
public void doTearDown(ServerState serverState) {
101+
public void doTearDown(ServerState serverState) throws Exception {
102102
dispose();
103103

104-
if (client != null) {
104+
try (ClickHouseResponse resp = client.connect(server).query("truncate table system.test_insert").execute()
105+
.get()) {
106+
107+
} finally {
105108
try {
106109
client.close();
107110
} catch (Exception e) {

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/Basic.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public class Basic extends DriverBenchmark {
1010
public int insertOneRandomNumber(DriverState state) throws Throwable {
1111
final int num = state.getRandomSample();
1212

13-
return executeInsert(state, "insert into test_insert(i) values(?)",
13+
return executeInsert(state, "insert into test_insert(i) values(?)", (p, v, l, i) -> p.setObject(i, v),
1414
Collections.enumeration(Collections.singletonList(new Object[] { num })));
1515
}
1616

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/DriverBenchmark.java

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,11 @@ public abstract class DriverBenchmark {
3232
private final int BATCH_SIZE = Integer.parseInt(System.getProperty("batchSize", "5000"));
3333
// fetch size for query
3434
private final int FETCH_SIZE = Integer.parseInt(System.getProperty("fetchSize", "1000"));
35+
// insert mode: 1) values; 2) table; 3) input
36+
private final String INSERT_MODE = System.getProperty("insertMode", "values").toLowerCase();
3537

36-
protected PreparedStatement setParameters(PreparedStatement s, Object... values) throws SQLException {
38+
protected PreparedStatement setParameters(PreparedStatement s, Object... values)
39+
throws SQLException {
3740
if (values != null && values.length > 0) {
3841
int index = 1;
3942
for (Object v : values) {
@@ -66,14 +69,22 @@ protected String replaceParameters(String sql, Object... values) {
6669
return sql;
6770
}
6871

69-
private int processBatch(Statement s, String sql, Enumeration<Object[]> generator) throws SQLException {
72+
private int processBatch(Statement s, String sql, SupplyValueFunction func, Enumeration<Object[]> generator)
73+
throws SQLException {
7074
int rows = 0;
7175
int counter = 0;
7276
PreparedStatement ps = s instanceof PreparedStatement ? (PreparedStatement) s : null;
7377
while (generator.hasMoreElements()) {
7478
Object[] values = generator.nextElement();
7579
if (ps != null) {
76-
setParameters(ps, values);
80+
int colIndex = 1;
81+
for (Object v : values) {
82+
if (colIndex == 1 && v instanceof String) {
83+
ps.setString(colIndex++, (String) v);
84+
} else {
85+
func.set(ps, v, rows, colIndex++);
86+
}
87+
}
7788

7889
if (BATCH_SIZE > 0) {
7990
ps.addBatch();
@@ -103,19 +114,26 @@ private int processBatch(Statement s, String sql, Enumeration<Object[]> generato
103114
return rows;
104115
}
105116

106-
protected int executeInsert(DriverState state, String sql, Enumeration<Object[]> generator) throws SQLException {
117+
protected int executeInsert(DriverState state, String sql, SupplyValueFunction func,
118+
Enumeration<Object[]> generator) throws SQLException {
107119
Objects.requireNonNull(generator);
108120

109121
final Connection conn = state.getConnection();
110122
int rows = 0;
111123

124+
if ("table".equals(INSERT_MODE)) {
125+
sql = sql.substring(0, sql.indexOf('\n') + 1);
126+
} else if ("input".equals(INSERT_MODE)) {
127+
sql = sql.substring(0, sql.indexOf('\n')).replaceFirst("--", "");
128+
}
129+
112130
if (state.usePreparedStatement()) {
113131
try (PreparedStatement s = conn.prepareStatement(sql)) {
114-
rows = processBatch(s, sql, generator);
132+
rows = processBatch(s, sql, func, generator);
115133
}
116134
} else {
117135
try (Statement s = conn.createStatement()) {
118-
rows = processBatch(s, sql, generator);
136+
rows = processBatch(s, sql, func, generator);
119137
}
120138
}
121139

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/DriverState.java

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import java.sql.SQLException;
66
import java.sql.Statement;
77
import java.util.Properties;
8+
import java.util.UUID;
89

910
import org.openjdk.jmh.annotations.Level;
1011
import org.openjdk.jmh.annotations.Param;
@@ -44,20 +45,22 @@ public void doSetup(ServerState serverState) throws Exception {
4445
JdbcDriver jdbcDriver = JdbcDriver.from(client);
4546

4647
String compression = String.valueOf(Boolean.parseBoolean(System.getProperty("compression", "true")));
48+
String additional = System.getProperty("additional", "");
4749

4850
try {
4951
driver = (java.sql.Driver) Class.forName(jdbcDriver.getClassName()).getDeclaredConstructor().newInstance();
5052
url = String.format(jdbcDriver.getUrlTemplate(), serverState.getHost(),
5153
serverState.getPort(jdbcDriver.getDefaultPort()), serverState.getDatabase(), serverState.getUser(),
52-
serverState.getPassword(), compression);
54+
serverState.getPassword(), compression, additional);
5355
// ClickHouseDefines.WRITE_COMPRESS = false;
5456
// ClickHouseDefines.READ_DECOMPRESS = Boolean.parseBoolean(compression);
5557
conn = driver.connect(url, new Properties());
5658

5759
try (Statement s = conn.createStatement()) {
60+
// s.execute("drop table if exists system.test_insert");
5861
s.execute("truncate table if exists system.test_insert");
5962
s.execute(
60-
"create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory");
63+
"create table if not exists system.test_insert(b String, i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory");
6164
}
6265

6366
if (!Constants.REUSE_CONNECTION.equalsIgnoreCase(connection)) {
@@ -138,4 +141,16 @@ public ConsumeValueFunction getConsumeFunction(ConsumeValueFunction defaultFunc)
138141
return defaultFunc;
139142
}
140143
}
144+
145+
public SupplyValueFunction getSupplyFunction(SupplyValueFunction defaultFunc) {
146+
if ("string".equals(type)) {
147+
return (p, v, l, i) -> p.setString(i, v != null ? v.toString() : null);
148+
} else if ("object".equals(type)) {
149+
return (p, v, l, i) -> p.setObject(i, v);
150+
} else if (defaultFunc == null) {
151+
return (p, v, l, i) -> p.setObject(i, v);
152+
} else {
153+
return defaultFunc;
154+
}
155+
}
141156
}

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/Insertion.java

Lines changed: 75 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,106 @@
11
package com.clickhouse.benchmark.jdbc;
22

3+
import java.sql.ResultSet;
4+
import java.sql.Statement;
35
import java.sql.Timestamp;
6+
import java.time.LocalDateTime;
7+
import java.time.ZoneOffset;
48
import java.util.Enumeration;
9+
import java.util.Locale;
10+
import java.util.UUID;
11+
512
import org.openjdk.jmh.annotations.Benchmark;
613

714
public class Insertion extends DriverBenchmark {
15+
private void checkResult(DriverState state, String batchId, int expectedRows, int actualResult) throws Exception {
16+
boolean isValid = actualResult == expectedRows;
17+
if (isValid) {
18+
try (Statement stmt = executeQuery(state,
19+
"select toInt32(count(1)) from system.test_insert where b=?", batchId)) {
20+
ResultSet rs = stmt.getResultSet();
21+
isValid = rs.next() && (actualResult = rs.getInt(1)) == expectedRows;
22+
}
23+
}
24+
if (!isValid) {
25+
throw new IllegalStateException(String.format(Locale.ROOT,
26+
"Expected %d rows being inserted but we got %d", expectedRows, actualResult));
27+
}
28+
}
29+
830
@Benchmark
9-
public int insert10kUInt64Rows(DriverState state) throws Throwable {
31+
public void insertInt64(DriverState state) throws Throwable {
1032
final int range = state.getRandomNumber();
1133
final int rows = state.getSampleSize() + range;
34+
final String batchId = UUID.randomUUID().toString();
35+
SupplyValueFunction func = state.getSupplyFunction((p, v, l, i) -> p.setLong(i, (long) v));
36+
int result = executeInsert(state,
37+
"insert into system.test_insert(b,i) -- select b,v from input('b String, v Int64')\nvalues(?,?)", func,
38+
new Enumeration<Object[]>() {
39+
int counter = 0;
1240

13-
return executeInsert(state, "insert into system.test_insert(i) values(?)", new Enumeration<Object[]>() {
14-
int counter = 0;
41+
@Override
42+
public boolean hasMoreElements() {
43+
return counter < rows;
44+
}
1545

16-
@Override
17-
public boolean hasMoreElements() {
18-
return counter < rows;
19-
}
46+
@Override
47+
public Object[] nextElement() {
48+
return new Object[] { batchId, (long) (range + (counter++)) };
49+
}
50+
});
2051

21-
@Override
22-
public Object[] nextElement() {
23-
return new Object[] { range + (counter++) };
24-
}
25-
});
52+
checkResult(state, batchId, rows, result);
2653
}
2754

2855
@Benchmark
29-
public int insert10kStringRows(DriverState state) throws Throwable {
56+
public void insertString(DriverState state) throws Throwable {
3057
final int range = state.getRandomNumber();
3158
final int rows = state.getSampleSize() + range;
59+
final String batchId = UUID.randomUUID().toString();
60+
SupplyValueFunction func = state.getSupplyFunction((p, v, l, i) -> p.setString(i, (String) v));
61+
int result = executeInsert(state,
62+
"insert into system.test_insert(b, s) -- select b, v from input('b String, v String')\nvalues(?, ?)",
63+
func, new Enumeration<Object[]>() {
64+
int counter = 0;
3265

33-
return executeInsert(state, "insert into system.test_insert(s) values(?)", new Enumeration<Object[]>() {
34-
int counter = 0;
66+
@Override
67+
public boolean hasMoreElements() {
68+
return counter < rows;
69+
}
3570

36-
@Override
37-
public boolean hasMoreElements() {
38-
return counter < rows;
39-
}
71+
@Override
72+
public Object[] nextElement() {
73+
return new Object[] { batchId, String.valueOf(range + (counter++)) };
74+
}
75+
});
4076

41-
@Override
42-
public Object[] nextElement() {
43-
return new Object[] { String.valueOf(range + (counter++)) };
44-
}
45-
});
77+
checkResult(state, batchId, rows, result);
4678
}
4779

4880
@Benchmark
49-
public int insert10kTimestampRows(DriverState state) throws Throwable {
81+
public void insertTimestamp(DriverState state) throws Throwable {
5082
final int range = state.getRandomNumber();
5183
final int rows = state.getSampleSize() + range;
84+
final String batchId = UUID.randomUUID().toString();
85+
SupplyValueFunction func = state
86+
.getSupplyFunction((p, v, l, i) -> p.setTimestamp(i, Timestamp.valueOf((LocalDateTime) v)));
87+
int result = executeInsert(state,
88+
"insert into system.test_insert(b,t) -- select b,v from input('b String,v DateTime32')\nvalues(?,?)",
89+
func, new Enumeration<Object[]>() {
90+
int counter = 0;
5291

53-
return executeInsert(state, "insert into system.test_insert(t) values(?)", new Enumeration<Object[]>() {
54-
int counter = 0;
92+
@Override
93+
public boolean hasMoreElements() {
94+
return counter < rows;
95+
}
5596

56-
@Override
57-
public boolean hasMoreElements() {
58-
return counter < rows;
59-
}
97+
@Override
98+
public Object[] nextElement() {
99+
return new Object[] { batchId,
100+
LocalDateTime.ofEpochSecond((long) range + (counter++), 0, ZoneOffset.UTC) };
101+
}
102+
});
60103

61-
@Override
62-
public Object[] nextElement() {
63-
return new Object[] { new Timestamp((long) range + (counter++)) };
64-
}
65-
});
104+
checkResult(state, batchId, rows, result);
66105
}
67106
}

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/JdbcDriver.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,41 +5,41 @@
55
public enum JdbcDriver {
66
// ClickHouse4j
77
Clickhouse4j("cc.blynk.clickhouse.ClickHouseDriver",
8-
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
8+
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
99
Constants.HTTP_PORT),
1010
// ClickHouse JDBC Driver
1111
ClickhouseHttpJdbc1("com.clickhouse.jdbc.ClickHouseDriver",
12-
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_URL_CONNECTION&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
12+
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_URL_CONNECTION&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
1313
Constants.HTTP_PORT),
1414
ClickhouseHttpJdbc2("com.clickhouse.jdbc.ClickHouseDriver",
15-
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_CLIENT&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
15+
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_CLIENT&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
1616
Constants.HTTP_PORT),
1717
ClickhouseGrpcJdbc("com.clickhouse.jdbc.ClickHouseDriver",
18-
"jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&max_inbound_message_size=2147483647&compress=%s",
18+
"jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&max_inbound_message_size=2147483647&compress=%s%s",
1919
Constants.GRPC_PORT),
2020
ClickhouseJdbc("ru.yandex.clickhouse.ClickHouseDriver",
21-
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
21+
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
2222
Constants.HTTP_PORT),
2323
// ClickHouse Native JDBC Driver
2424
ClickhouseNativeJdbc("com.github.housepower.jdbc.ClickHouseDriver",
25-
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
25+
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
2626
Constants.NATIVE_PORT),
2727

2828
// MariaDB Java Client
2929
MariadbJavaClient("org.mariadb.jdbc.Driver",
3030
"jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useServerPrepStmts=false&useCompression=%s"
31-
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC",
31+
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC%s",
3232
Constants.MYSQL_PORT),
3333

3434
// MySQL Connector/J
3535
MysqlConnectorJava("com.mysql.cj.jdbc.Driver",
3636
"jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useServerPrepStmts=false"
37-
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC&useCompression=%s",
37+
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC&useCompression=%S%s",
3838
Constants.MYSQL_PORT),
3939

4040
// PostgreSQL JDBC Driver
4141
PostgresqlJdbc("org.postgresql.Driver",
42-
"jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple&compress=%s",
42+
"jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple&compress=%s%s",
4343
Constants.POSTGRESQL_PORT);
4444

4545
private final String className;

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/jdbc/SupplyValueFunction.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package com.clickhouse.benchmark.jdbc;
2+
3+
import java.sql.PreparedStatement;
4+
import java.sql.SQLException;
5+
6+
@FunctionalInterface
7+
public interface SupplyValueFunction {
8+
void set(PreparedStatement ps, Object value, int rowIndex, int columnIndex) throws SQLException;
9+
}

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/FactoryBenchmark.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.clickhouse.benchmark.BaseState;
3131
import com.clickhouse.client.ClickHouseEnum;
3232
import com.clickhouse.client.ClickHouseValue;
33+
import com.clickhouse.client.ClickHouseValues;
3334
import com.clickhouse.client.data.ClickHouseBigDecimalValue;
3435
import com.clickhouse.client.data.ClickHouseBigIntegerValue;
3536
import com.clickhouse.client.data.ClickHouseByteValue;
@@ -89,8 +90,8 @@ public void setupSamples() {
8990

9091
// add(map, list, Object[].class, () -> ClickHouseArrayValue.of((Object[]) o));
9192
add(map, list, LocalDate.class, () -> ClickHouseDateValue.ofNull());
92-
add(map, list, LocalTime.class, () -> ClickHouseDateTimeValue.ofNull(0));
93-
add(map, list, LocalDateTime.class, () -> ClickHouseDateTimeValue.ofNull(0));
93+
add(map, list, LocalTime.class, () -> ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE));
94+
add(map, list, LocalDateTime.class, () -> ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE));
9495
add(map, list, String.class, () -> ClickHouseStringValue.ofNull());
9596

9697
mappings = Collections.unmodifiableMap(map);
@@ -130,9 +131,9 @@ ClickHouseValue newValue(Class<?> clazz) {
130131
} else if (LocalDate.class.equals(clazz)) {
131132
return ClickHouseDateValue.ofNull();
132133
} else if (LocalTime.class.equals(clazz)) {
133-
return ClickHouseDateTimeValue.ofNull(0);
134+
return ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE);
134135
} else if (LocalDateTime.class.equals(clazz)) {
135-
return ClickHouseDateTimeValue.ofNull(0);
136+
return ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE);
136137
} else if (String.class.equals(clazz)) {
137138
return ClickHouseStringValue.ofNull();
138139
}

0 commit comments

Comments
 (0)