Skip to content

Commit 9267440

Browse files
zhicwuzhicwu
authored
Merge pull request #596 from zhicwu/roaringbitmap
RoaringBitmap support
2 parents a90da54 + bbfca81 commit 9267440

34 files changed

+1509
-305
lines changed

‎.github/workflows/analysis.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ jobs:
6161
# added,diff_context,file,nofilter
6262
filter_mode: 'added'
6363
if: github.event_name == 'pull_request_target' || github.event.inputs.pr != ''
64+
continue-on-error: true
6465
- name: Update sonar config
6566
run: |
6667
sed -i -e 's|^\(.*<sonar.projectKey>\).*\(</sonar.projectKey>\)$|\1ClickHouse_clickhouse-jdbc\2|' \
@@ -73,3 +74,4 @@ jobs:
7374
run: |
7475
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
7576
mvn -q --batch-mode -Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
77+
continue-on-error: true

‎.github/workflows/build.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ jobs:
3434
matrix:
3535
java: [8, 11, 15]
3636
# most recent LTS releases as well as latest stable builds
37-
clickhouse: ["20.8", "20.10", "20.12", "21.2", "latest"]
37+
clickhouse: ["20.8", "21.3", "latest"]
38+
fail-fast: false
3839
name: Build using JDK ${{ matrix.java }} against ClickHouse ${{ matrix.clickhouse }}
3940
steps:
4041
- name: Check out Git repository

‎.github/workflows/timezone.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: TimeZone Test
1+
name: TimeZone
22

33
on:
44
push:

‎clickhouse-benchmark/pom.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@
1818

1919
<properties>
2020
<clickhouse4j-driver.version>1.4.4</clickhouse4j-driver.version>
21-
<native-driver.version>2.5.3</native-driver.version>
2221
<mariadb-driver.version>2.7.2</mariadb-driver.version>
2322
<mysql-driver.version>8.0.23</mysql-driver.version>
24-
<native-driver.version>2.5.3</native-driver.version>
23+
<native-driver.version>2.5.4</native-driver.version>
2524
<testcontainers.version>1.15.2</testcontainers.version>
2625
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
2726
<jmh.version>1.27</jmh.version>

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Basic.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import java.sql.Statement;
55
import java.util.Collections;
66
import java.util.Random;
7-
87
import org.openjdk.jmh.annotations.Benchmark;
98

109
public class Basic extends JdbcBenchmark {

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ClientState.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import java.sql.SQLException;
55
import java.sql.Statement;
66
import java.util.Properties;
7-
87
import org.openjdk.jmh.annotations.Level;
98
import org.openjdk.jmh.annotations.Param;
109
import org.openjdk.jmh.annotations.Scope;

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Constants.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package tech.clickhouse.benchmark;
22

3+
/**
4+
* Constant interface.
5+
*/
36
public interface Constants {
47
public static final String CLICKHOUSE_DRIVER = "clickhouse-jdbc";
58

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Insertion.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
// import java.util.Collections;
55
import java.util.Enumeration;
66
import java.util.Random;
7-
87
import org.openjdk.jmh.annotations.Benchmark;
98

109
public class Insertion extends JdbcBenchmark {

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcBenchmark.java

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,25 @@
11
package tech.clickhouse.benchmark;
22

3-
import org.openjdk.jmh.annotations.*;
4-
53
import java.sql.Connection;
64
import java.sql.PreparedStatement;
75
import java.sql.SQLException;
86
import java.sql.Statement;
97
import java.util.Enumeration;
108
import java.util.Objects;
119
import java.util.concurrent.TimeUnit;
12-
10+
import org.openjdk.jmh.annotations.BenchmarkMode;
11+
import org.openjdk.jmh.annotations.Fork;
12+
import org.openjdk.jmh.annotations.Measurement;
13+
import org.openjdk.jmh.annotations.Mode;
14+
import org.openjdk.jmh.annotations.OutputTimeUnit;
15+
import org.openjdk.jmh.annotations.Scope;
16+
import org.openjdk.jmh.annotations.State;
17+
import org.openjdk.jmh.annotations.Threads;
18+
import org.openjdk.jmh.annotations.Warmup;
19+
20+
/**
21+
* Base class for JDBC driver benchmarking.
22+
*/
1323
@State(Scope.Benchmark)
1424
@Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1)
1525
@Measurement(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1)

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/JdbcDriver.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ public enum JdbcDriver {
1616

1717
// MariaDB Java Client
1818
MariadbJavaClient("org.mariadb.jdbc.Driver",
19-
"jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC",
19+
"jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false"
20+
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC",
2021
Constants.MYSQL_PORT),
2122

2223
// MySQL Connector/J
2324
MysqlConnectorJava("com.mysql.cj.jdbc.Driver",
24-
"jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC",
25+
"jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useCompression=true&useServerPrepStmts=false"
26+
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC",
2527
Constants.MYSQL_PORT);
2628

2729
private final String className;

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/Query.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import java.sql.Statement;
55
import java.sql.Timestamp;
66
import java.util.Random;
7-
87
import org.openjdk.jmh.annotations.Benchmark;
98

109
public class Query extends JdbcBenchmark {

‎clickhouse-benchmark/src/main/java/tech/clickhouse/benchmark/ServerState.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
package tech.clickhouse.benchmark;
22

3+
import static java.time.temporal.ChronoUnit.SECONDS;
4+
35
import java.net.Inet4Address;
46
import java.net.InetAddress;
57
import java.net.InterfaceAddress;
68
import java.net.NetworkInterface;
79
import java.time.Duration;
810
import java.util.Enumeration;
9-
1011
import org.openjdk.jmh.annotations.Level;
1112
import org.openjdk.jmh.annotations.Scope;
1213
import org.openjdk.jmh.annotations.Setup;
@@ -16,8 +17,6 @@
1617
import org.testcontainers.containers.wait.strategy.Wait;
1718
import org.testcontainers.images.builder.ImageFromDockerfile;
1819

19-
import static java.time.temporal.ChronoUnit.SECONDS;
20-
2120
@State(Scope.Benchmark)
2221
public class ServerState {
2322
static String getLocalIpAddress() {

‎clickhouse-jdbc/pom.xml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@
7272
<groupId>com.fasterxml.jackson.core</groupId>
7373
<artifactId>jackson-databind</artifactId>
7474
</dependency>
75+
<dependency>
76+
<groupId>com.github.RoaringBitmap</groupId>
77+
<artifactId>RoaringBitmap</artifactId>
78+
</dependency>
7579
<dependency>
7680
<groupId>org.slf4j</groupId>
7781
<artifactId>slf4j-api</artifactId>
@@ -198,6 +202,10 @@
198202
<pattern>net.jpountz</pattern>
199203
<shadedPattern>${shade.base}.jpountz</shadedPattern>
200204
</relocation>
205+
<relocation>
206+
<pattern>org.roaringbitmap</pattern>
207+
<shadedPattern>${shade.base}.bitmap</shadedPattern>
208+
</relocation>
201209
<relocation>
202210
<pattern>org.slf4j</pattern>
203211
<shadedPattern>${shade.base}.slf4j</shadedPattern>

‎clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ByteFragment.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public ByteFragment(byte[] buf, int start, int len) {
1919
}
2020

2121
public static ByteFragment fromString(String str) {
22+
// https://bugs.openjdk.java.net/browse/JDK-6219899
2223
byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
2324
return new ByteFragment(bytes, 0, bytes.length);
2425
}

‎clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/response/ClickHouseColumnInfo.java

Lines changed: 83 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package ru.yandex.clickhouse.response;
22

33
import java.util.TimeZone;
4-
54
import ru.yandex.clickhouse.domain.ClickHouseDataType;
65

6+
/**
7+
* This class represents a column defined in database.
8+
*/
79
public final class ClickHouseColumnInfo {
810

911
private static final String KEYWORD_NULLABLE = "Nullable";
@@ -22,12 +24,21 @@ public final class ClickHouseColumnInfo {
2224
private int scale;
2325
private ClickHouseColumnInfo keyInfo;
2426
private ClickHouseColumnInfo valueInfo;
27+
private String functionName;
2528

2629
@Deprecated
2730
public static ClickHouseColumnInfo parse(String typeInfo, String columnName) {
2831
return parse(typeInfo, columnName, null);
2932
}
3033

34+
/**
35+
* Parse given type string.
36+
*
37+
* @param typeInfo type defined in database
38+
* @param columnName column name
39+
* @param serverTimeZone server time zone
40+
* @return parsed type
41+
*/
3142
public static ClickHouseColumnInfo parse(String typeInfo, String columnName, TimeZone serverTimeZone) {
3243
ClickHouseColumnInfo column = new ClickHouseColumnInfo(typeInfo, columnName);
3344
int currIdx = 0;
@@ -61,68 +72,74 @@ public static ClickHouseColumnInfo parse(String typeInfo, String columnName, Tim
6172
column.scale = dataType.getDefaultScale();
6273
column.timeZone = serverTimeZone;
6374
currIdx = endIdx;
64-
if (endIdx == typeInfo.length()
65-
|| !typeInfo.startsWith("(", currIdx))
66-
{
75+
if (endIdx == typeInfo.length() || !typeInfo.startsWith("(", currIdx)) {
6776
return column;
6877
}
6978

7079
switch (dataType) {
71-
case DateTime :
72-
String[] argsDT = splitArgs(typeInfo, currIdx);
73-
if (argsDT.length == 2) { // same as DateTime64
74-
column.scale = Integer.parseInt(argsDT[0]);
75-
column.timeZone = TimeZone.getTimeZone(argsDT[1].replace("'", ""));
76-
} else if (argsDT.length == 1) { // same as DateTime32
77-
// unfortunately this will fall back to GMT if the time zone
78-
// cannot be resolved
79-
TimeZone tz = TimeZone.getTimeZone(argsDT[0].replace("'", ""));
80-
column.timeZone = tz;
81-
}
82-
break;
83-
case DateTime32:
84-
String[] argsD32 = splitArgs(typeInfo, currIdx);
85-
if (argsD32.length == 1) {
86-
// unfortunately this will fall back to GMT if the time zone
87-
// cannot be resolved
88-
TimeZone tz = TimeZone.getTimeZone(argsD32[0].replace("'", ""));
89-
column.timeZone = tz;
90-
}
91-
break;
92-
case DateTime64:
93-
String[] argsD64 = splitArgs(typeInfo, currIdx);
94-
if (argsD64.length == 2) {
95-
column.scale = Integer.parseInt(argsD64[0]);
96-
column.timeZone = TimeZone.getTimeZone(argsD64[1].replace("'", ""));
97-
}
98-
break;
99-
case Decimal :
100-
String[] argsDecimal = splitArgs(typeInfo, currIdx);
101-
if (argsDecimal.length == 2) {
102-
column.precision = Integer.parseInt(argsDecimal[0]);
103-
column.scale = Integer.parseInt(argsDecimal[1]);
104-
}
105-
break;
106-
case Decimal32 :
107-
case Decimal64 :
108-
case Decimal128 :
109-
case Decimal256 :
110-
String[] argsScale = splitArgs(typeInfo, currIdx);
111-
column.scale = Integer.parseInt(argsScale[0]);
112-
break;
113-
case FixedString :
114-
String[] argsPrecision = splitArgs(typeInfo, currIdx);
115-
column.precision = Integer.parseInt(argsPrecision[0]);
116-
break;
117-
case Map:
118-
String[] argsMap = splitArgs(typeInfo, currIdx);
119-
if (argsMap.length == 2) {
120-
column.keyInfo = ClickHouseColumnInfo.parse(argsMap[0], columnName + "Key", serverTimeZone);
121-
column.valueInfo = ClickHouseColumnInfo.parse(argsMap[1], columnName + "Value", serverTimeZone);
122-
}
123-
break;
124-
default :
125-
break;
80+
case AggregateFunction :
81+
String[] argsAf = splitArgs(typeInfo, currIdx);
82+
column.functionName = argsAf[0];
83+
column.arrayBaseType = ClickHouseDataType.Unknown;
84+
if (argsAf.length == 2) {
85+
column.arrayBaseType = ClickHouseDataType.fromTypeString(argsAf[1]);
86+
}
87+
break;
88+
case DateTime :
89+
String[] argsDt = splitArgs(typeInfo, currIdx);
90+
if (argsDt.length == 2) { // same as DateTime64
91+
column.scale = Integer.parseInt(argsDt[0]);
92+
column.timeZone = TimeZone.getTimeZone(argsDt[1].replace("'", ""));
93+
} else if (argsDt.length == 1) { // same as DateTime32
94+
// unfortunately this will fall back to GMT if the time zone
95+
// cannot be resolved
96+
TimeZone tz = TimeZone.getTimeZone(argsDt[0].replace("'", ""));
97+
column.timeZone = tz;
98+
}
99+
break;
100+
case DateTime32:
101+
String[] argsD32 = splitArgs(typeInfo, currIdx);
102+
if (argsD32.length == 1) {
103+
// unfortunately this will fall back to GMT if the time zone
104+
// cannot be resolved
105+
TimeZone tz = TimeZone.getTimeZone(argsD32[0].replace("'", ""));
106+
column.timeZone = tz;
107+
}
108+
break;
109+
case DateTime64:
110+
String[] argsD64 = splitArgs(typeInfo, currIdx);
111+
if (argsD64.length == 2) {
112+
column.scale = Integer.parseInt(argsD64[0]);
113+
column.timeZone = TimeZone.getTimeZone(argsD64[1].replace("'", ""));
114+
}
115+
break;
116+
case Decimal :
117+
String[] argsDecimal = splitArgs(typeInfo, currIdx);
118+
if (argsDecimal.length == 2) {
119+
column.precision = Integer.parseInt(argsDecimal[0]);
120+
column.scale = Integer.parseInt(argsDecimal[1]);
121+
}
122+
break;
123+
case Decimal32 :
124+
case Decimal64 :
125+
case Decimal128 :
126+
case Decimal256 :
127+
String[] argsScale = splitArgs(typeInfo, currIdx);
128+
column.scale = Integer.parseInt(argsScale[0]);
129+
break;
130+
case FixedString :
131+
String[] argsPrecision = splitArgs(typeInfo, currIdx);
132+
column.precision = Integer.parseInt(argsPrecision[0]);
133+
break;
134+
case Map:
135+
String[] argsMap = splitArgs(typeInfo, currIdx);
136+
if (argsMap.length == 2) {
137+
column.keyInfo = ClickHouseColumnInfo.parse(argsMap[0], columnName + "Key", serverTimeZone);
138+
column.valueInfo = ClickHouseColumnInfo.parse(argsMap[1], columnName + "Value", serverTimeZone);
139+
}
140+
break;
141+
default:
142+
break;
126143
}
127144

128145
return column;
@@ -153,8 +170,9 @@ public String getOriginalTypeName() {
153170
}
154171

155172
/**
156-
* @return the type name returned from the database, without modifiers, i.e.
157-
* Nullable or LowCardinality
173+
* Get the type name returned from the database, without modifiers, i.e. Nullable or LowCardinality.
174+
*
175+
* @return the type name returned from the database
158176
*/
159177
public String getCleanTypeName() {
160178
if (!nullable && !lowCardinality) {
@@ -226,4 +244,8 @@ public ClickHouseColumnInfo getKeyInfo() {
226244
public ClickHouseColumnInfo getValueInfo() {
227245
return this.valueInfo;
228246
}
247+
248+
public String getFunctionName() {
249+
return this.functionName;
250+
}
229251
}

0 commit comments

Comments
 (0)