Skip to content

Commit c98c084

Browse files
zhicwuzhicwu
authored
Merge pull request #933 from ClickHouse/develop
Merge patch9
2 parents f9f9683 + afa6cdd commit c98c084

File tree

79 files changed

+3545
-573
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+3545
-573
lines changed

‎.github/workflows/build.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ jobs:
4242
- clickhouse: "21.8"
4343
protocol: grpc
4444
fail-fast: false
45+
timeout-minutes: 45
4546
name: Build against ClickHouse ${{ matrix.clickhouse }} (${{ matrix.protocol }})
4647
steps:
4748
- name: Check out Git repository
@@ -90,3 +91,11 @@ jobs:
9091
- name: Build
9192
run: |
9293
mvn --batch-mode --update-snapshots -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} verify
94+
- name: Upload test results
95+
uses: actions/upload-artifact@v2
96+
if: failure()
97+
with:
98+
name: result ${{ github.job }}
99+
path: |
100+
**/target/failsafe-reports
101+
**/target/surefire-reports

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ Note: in general, the new driver(v0.3.2) is a few times faster with less memory
115115
<groupId>com.clickhouse</groupId>
116116
<!-- or clickhouse-grpc-client if you prefer gRPC -->
117117
<artifactId>clickhouse-http-client</artifactId>
118-
<version>0.3.2-patch8</version>
118+
<version>0.3.2-patch9</version>
119119
</dependency>
120120
```
121121

@@ -151,7 +151,7 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol);
151151
<!-- will stop using ru.yandex.clickhouse starting from 0.4.0 -->
152152
<groupId>com.clickhouse</groupId>
153153
<artifactId>clickhouse-jdbc</artifactId>
154-
<version>0.3.2-patch8</version>
154+
<version>0.3.2-patch9</version>
155155
<!-- below is only needed when all you want is a shaded jar -->
156156
<classifier>http</classifier>
157157
<exclusions>

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/client/ClientState.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import com.clickhouse.client.ClickHouseClient;
1616
import com.clickhouse.client.ClickHouseClientBuilder;
1717
import com.clickhouse.client.ClickHouseCredentials;
18+
import com.clickhouse.client.ClickHouseException;
1819
import com.clickhouse.client.ClickHouseCompression;
1920
import com.clickhouse.client.ClickHouseFormat;
2021
import com.clickhouse.client.ClickHouseNode;
@@ -79,7 +80,7 @@ private ClickHouseClient createClient() {
7980
}
8081

8182
@Setup(Level.Trial)
82-
public void doSetup(ServerState serverState) throws Exception {
83+
public void doSetup(ServerState serverState) throws ClickHouseException {
8384
server = ClickHouseNode.builder().host(serverState.getHost()).port(ClickHouseProtocol.valueOf(protocol))
8485
.database(serverState.getDatabase())
8586
.credentials(
@@ -91,18 +92,18 @@ public void doSetup(ServerState serverState) throws Exception {
9192
"create table if not exists system.test_insert(id String, i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory" };
9293

9394
for (String sql : sqls) {
94-
try (ClickHouseResponse resp = client.connect(server).query(sql).execute().get()) {
95+
try (ClickHouseResponse resp = client.connect(server).query(sql).executeAndWait()) {
9596

9697
}
9798
}
9899
}
99100

100101
@TearDown(Level.Trial)
101-
public void doTearDown(ServerState serverState) throws Exception {
102+
public void doTearDown(ServerState serverState) throws ClickHouseException {
102103
dispose();
103104

104-
try (ClickHouseResponse resp = client.connect(server).query("truncate table system.test_insert").execute()
105-
.get()) {
105+
try (ClickHouseResponse resp = client.connect(server).query("truncate table system.test_insert")
106+
.executeAndWait()) {
106107

107108
} finally {
108109
try {

‎clickhouse-benchmark/src/main/java/com/clickhouse/benchmark/misc/StreamBenchmark.java

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package com.clickhouse.benchmark.misc;
2+
3+
import java.io.ByteArrayInputStream;
4+
import java.io.ByteArrayOutputStream;
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.io.Serializable;
8+
import java.util.Arrays;
9+
import java.util.HashMap;
10+
import java.util.Map;
11+
import java.util.Random;
12+
import java.util.concurrent.TimeUnit;
13+
14+
import com.clickhouse.benchmark.BaseState;
15+
import com.clickhouse.client.ClickHouseClient;
16+
import com.clickhouse.client.ClickHouseConfig;
17+
import com.clickhouse.client.ClickHouseInputStream;
18+
import com.clickhouse.client.ClickHouseOutputStream;
19+
import com.clickhouse.client.config.ClickHouseBufferingMode;
20+
import com.clickhouse.client.config.ClickHouseClientOption;
21+
import com.clickhouse.client.config.ClickHouseOption;
22+
23+
import org.openjdk.jmh.annotations.Benchmark;
24+
import org.openjdk.jmh.annotations.BenchmarkMode;
25+
import org.openjdk.jmh.annotations.Fork;
26+
import org.openjdk.jmh.annotations.Level;
27+
import org.openjdk.jmh.annotations.Measurement;
28+
import org.openjdk.jmh.annotations.Mode;
29+
import org.openjdk.jmh.annotations.OutputTimeUnit;
30+
import org.openjdk.jmh.annotations.Scope;
31+
import org.openjdk.jmh.annotations.Setup;
32+
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.Threads;
34+
import org.openjdk.jmh.annotations.Warmup;
35+
import org.openjdk.jmh.infra.Blackhole;
36+
37+
/**
38+
* Blocking:
39+
* Benchmark Mode Cnt Score Error Units
40+
* StreamBenchmark.async thrpt 20 1.574 ? 0.039 ops/s
41+
* StreamBenchmark.jdk thrpt 20 4281.206 ? 91.983 ops/s
42+
* StreamBenchmark.piped thrpt 20 3913.994 ? 142.566 ops/s
43+
* StreamBenchmark.wrapped thrpt 20 3939.248 ? 54.868 ops/s
44+
*
45+
* Non-blocking:
46+
*
47+
*/
48+
@State(Scope.Benchmark)
49+
@Warmup(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1)
50+
@Measurement(iterations = 10, timeUnit = TimeUnit.SECONDS, time = 1)
51+
@Fork(value = 2)
52+
@Threads(value = -1)
53+
@BenchmarkMode(Mode.Throughput)
54+
@OutputTimeUnit(TimeUnit.SECONDS)
55+
public class StreamBenchmark {
56+
@State(Scope.Thread)
57+
public static class StreamState extends BaseState {
58+
public int bufferSize;
59+
public int samples;
60+
61+
public byte[] bytes;
62+
public ClickHouseConfig config;
63+
64+
@Setup(Level.Trial)
65+
public void setupSamples() {
66+
bufferSize = Integer.getInteger("buffer", (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue());
67+
samples = Integer.getInteger("samples", 500000);
68+
69+
bytes = new byte[samples];
70+
71+
Map<ClickHouseOption, Serializable> options = new HashMap<>();
72+
options.put(ClickHouseClientOption.ASYNC, Boolean.parseBoolean(System.getProperty("async", "true")));
73+
options.put(ClickHouseClientOption.REQUEST_BUFFERING, ClickHouseBufferingMode.valueOf(
74+
System.getProperty("mode", ClickHouseClientOption.REQUEST_BUFFERING.getDefaultValue().toString())
75+
.toUpperCase()));
76+
options.put(ClickHouseClientOption.BUFFER_SIZE, bufferSize);
77+
options.put(ClickHouseClientOption.MAX_QUEUED_BUFFERS,
78+
Integer.getInteger("queue", (int) ClickHouseClientOption.MAX_QUEUED_BUFFERS.getDefaultValue()));
79+
options.put(ClickHouseClientOption.COMPRESS, Boolean.parseBoolean(System.getProperty("compress", "false")));
80+
options.put(ClickHouseClientOption.DECOMPRESS,
81+
Boolean.parseBoolean(System.getProperty("compress", "false")));
82+
options.put(ClickHouseClientOption.USE_BLOCKING_QUEUE,
83+
Boolean.parseBoolean(System.getProperty("blocking", "true")));
84+
config = new ClickHouseConfig(options, null, null, null);
85+
}
86+
87+
@Setup(Level.Iteration)
88+
public void initStream() {
89+
new Random().nextBytes(bytes);
90+
}
91+
}
92+
93+
@Benchmark
94+
public void classic(StreamState state, Blackhole consumer) throws IOException {
95+
int size = state.bufferSize;
96+
byte[] buffer = new byte[size];
97+
int count = 0;
98+
ByteArrayOutputStream out = new ByteArrayOutputStream(state.samples);
99+
try (InputStream in = new ByteArrayInputStream(state.bytes)) {
100+
int read = 0;
101+
while ((read = in.read(buffer, 0, size)) > 0) {
102+
out.write(buffer, 0, read);
103+
count += read;
104+
}
105+
if (count != state.samples) {
106+
throw new IllegalStateException(String.format("Expect %d bytes but got %d", size, count));
107+
}
108+
out.flush();
109+
out.close();
110+
}
111+
if (!Arrays.equals(state.bytes, out.toByteArray())) {
112+
throw new IllegalStateException("Incorrect result");
113+
}
114+
}
115+
116+
@Benchmark
117+
public void piped(StreamState state, Blackhole consumer) throws IOException {
118+
int size = state.bufferSize;
119+
long count = 0;
120+
ByteArrayOutputStream out = new ByteArrayOutputStream(state.samples);
121+
try (InputStream in = new ByteArrayInputStream(state.bytes)) {
122+
if ((count = ClickHouseInputStream.pipe(in, out, size)) != state.samples) {
123+
throw new IllegalStateException(String.format("Expect %d bytes but got %d", size, count));
124+
}
125+
out.flush();
126+
out.close();
127+
}
128+
if (!Arrays.equals(state.bytes, out.toByteArray())) {
129+
throw new IllegalStateException("Incorrect result");
130+
}
131+
}
132+
133+
@Benchmark
134+
public void wrapped(StreamState state, Blackhole consumer) throws IOException {
135+
int size = state.bufferSize;
136+
long count = 0;
137+
ByteArrayOutputStream bao = new ByteArrayOutputStream(state.samples);
138+
try (ClickHouseInputStream in = ClickHouseInputStream.of(new ByteArrayInputStream(state.bytes), size);
139+
ClickHouseOutputStream out = ClickHouseOutputStream.of(bao, size)) {
140+
if ((count = in.pipe(out)) != state.samples) {
141+
throw new IllegalStateException(String.format("Expect %d bytes but got %d", size, count));
142+
}
143+
out.flush();
144+
}
145+
if (!Arrays.equals(state.bytes, bao.toByteArray())) {
146+
throw new IllegalStateException("Incorrect result");
147+
}
148+
}
149+
150+
@Benchmark
151+
public void async(StreamState state, Blackhole consumer) throws IOException {
152+
int size = state.bufferSize;
153+
long count = 0;
154+
ByteArrayOutputStream bao = new ByteArrayOutputStream(state.samples);
155+
try (ClickHouseInputStream in = ClickHouseInputStream.of(new ByteArrayInputStream(state.bytes), size);
156+
ClickHouseOutputStream out = ClickHouseClient.getAsyncRequestOutputStream(state.config, bao, null)) {
157+
if ((count = in.pipe(out)) != state.samples) {
158+
throw new IllegalStateException(String.format("Expect %d bytes but got %d", size, count));
159+
}
160+
out.flush();
161+
}
162+
if (!Arrays.equals(state.bytes, bao.toByteArray())) {
163+
throw new IllegalStateException("Incorrect result");
164+
}
165+
}
166+
}

‎clickhouse-client/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so
99
<dependency>
1010
<groupId>com.clickhouse</groupId>
1111
<artifactId>clickhouse-http-client</artifactId>
12-
<version>0.3.2-patch8</version>
12+
<version>0.3.2-patch9</version>
1313
</dependency>
1414
```
1515

0 commit comments

Comments
 (0)