Skip to content

Commit 92b025a

Browse files
zhicwuzhicwu
authored
Merge pull request #1291 from zhicwu/main
New JDBC option to support INFILE and OUTFILE
2 parents c079c99 + 4979706 commit 92b025a

File tree

22 files changed

+1019
-170
lines changed

22 files changed

+1019
-170
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
## 0.4.2
22

3+
### Breaking Changes
4+
* ClickHouseSqlStatement and *ParserHandler in JDBC driver were refactored to support `compression` and `infile` in insert statement.
5+
36
### New Features
47
* centralized configuration for JDBC driver using custom server setting `custom_jdbc_config`.
58
* support `BEGIN TRANSACTION`, `COMMIT`, and `ROLLBACK` in JDBC driver. [#975](https://github.com/ClickHouse/clickhouse-java/issues/975)
69
* new options for JDBC driver
710
* databaseTerm(catalog or schema, defaults to schema) [#1273](https://github.com/ClickHouse/clickhouse-java/issues/1273)
811
* externalDatabase(true or false, defaults to true) [#1245](https://github.com/ClickHouse/clickhouse-java/issues/1245)
12+
* localFile(true of false, defaults to false) - whether to use local file for INFILE and OUTFILE or not
913

1014
### Bug Fixes
1115
* error while converting Nested values to Java maps.

‎clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -403,8 +403,10 @@ protected ClickHouseRequest(ClickHouseClient client, Function<ClickHouseNodeSele
403403
this.txRef = new AtomicReference<>(null);
404404

405405
this.externalTables = new LinkedList<>();
406-
this.options = ClickHouseFreezableMap.of(new HashMap<>());
407-
this.settings = ClickHouseFreezableMap.of(new LinkedHashMap<>(client.getConfig().getCustomSettings()));
406+
// TODO configurable whitelist? maybe later
407+
this.options = ClickHouseFreezableMap.of(new HashMap<>(), ClickHouseClientOption.SESSION_ID);
408+
this.settings = ClickHouseFreezableMap.of(new LinkedHashMap<>(client.getConfig().getCustomSettings()),
409+
ClickHouseTransaction.SETTING_IMPLICIT_TRANSACTION);
408410
options(options);
409411

410412
this.namedParameters = new HashMap<>();

‎clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponseSummary.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,16 @@ public long getWrittenBytes() {
6666
return written_bytes;
6767
}
6868

69+
public Progress add(Progress progress) {
70+
if (progress == null) {
71+
return this;
72+
}
73+
74+
return new Progress(read_rows + progress.read_rows, read_bytes + progress.read_bytes,
75+
total_rows_to_read + progress.total_rows_to_read, written_rows + progress.written_rows,
76+
written_bytes + progress.written_bytes);
77+
}
78+
6979
public boolean isEmpty() {
7080
return read_rows == 0L && read_bytes == 0L && total_rows_to_read == 0L && written_rows == 0L
7181
&& written_bytes == 0L;
@@ -124,6 +134,15 @@ public long getRowsBeforeLimit() {
124134
return rows_before_limit;
125135
}
126136

137+
public Statistics add(Statistics stats) {
138+
if (stats == null) {
139+
return this;
140+
}
141+
142+
return new Statistics(rows + stats.rows, blocks + stats.blocks, allocated_bytes + stats.allocated_bytes,
143+
applied_limit || stats.applied_limit, rows_before_limit + stats.rows_before_limit);
144+
}
145+
127146
public boolean isEmpty() {
128147
return rows == 0L && blocks == 0L && allocated_bytes == 0L && !applied_limit && rows_before_limit == 0L;
129148
}
@@ -202,6 +221,16 @@ public void update(Progress progress) {
202221
}
203222
}
204223

224+
public void add(Progress progress) {
225+
if (sealed) {
226+
throw new IllegalStateException(ERROR_CANNOT_UPDATE);
227+
}
228+
229+
Progress current = this.progress.get();
230+
this.progress.set(current.add(progress));
231+
this.updates.incrementAndGet();
232+
}
233+
205234
public void update(Statistics stats) {
206235
if (sealed) {
207236
throw new IllegalStateException(ERROR_CANNOT_UPDATE);
@@ -212,6 +241,24 @@ public void update(Statistics stats) {
212241
}
213242
}
214243

244+
public void add(Statistics stats) {
245+
if (sealed) {
246+
throw new IllegalStateException(ERROR_CANNOT_UPDATE);
247+
}
248+
249+
Statistics current = this.stats.get();
250+
this.stats.set(current.add(stats));
251+
}
252+
253+
public void add(ClickHouseResponseSummary summary) {
254+
if (summary == null) {
255+
return;
256+
}
257+
258+
add(summary.getProgress());
259+
add(summary.getStatistics());
260+
}
261+
215262
/**
216263
* Gets current progress of the query.
217264
*

‎clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSimpleResponse.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,15 @@ public static ClickHouseResponse of(ClickHouseConfig config, List<ClickHouseColu
4141
*/
4242
public static ClickHouseResponse of(ClickHouseConfig config, List<ClickHouseColumn> columns, Object[][] values,
4343
ClickHouseResponseSummary summary) {
44-
if (columns == null || columns.isEmpty()) {
44+
if (columns == null) {
45+
columns = Collections.emptyList();
46+
}
47+
if (columns.isEmpty() && (summary == null || summary == ClickHouseResponseSummary.EMPTY)) {
4548
return ClickHouseResponse.EMPTY;
4649
}
4750

4851
int size = columns.size();
4952
int len = values != null ? values.length : 0;
50-
5153
ClickHouseValue[][] wrappedValues = new ClickHouseValue[len][];
5254
if (len > 0) {
5355
ClickHouseValue[] templates = new ClickHouseValue[size];

‎clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseResponseSummaryTest.java

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package com.clickhouse.client;
2+
3+
import org.testng.Assert;
4+
import org.testng.annotations.Test;
5+
6+
import com.clickhouse.client.ClickHouseResponseSummary.Progress;
7+
import com.clickhouse.client.ClickHouseResponseSummary.Statistics;
8+
9+
public class ClickHouseResponseSummaryTest {
10+
@Test(groups = { "unit" })
11+
public void testConsutrctor() {
12+
ClickHouseResponseSummary summary = new ClickHouseResponseSummary(null, null);
13+
Assert.assertNotNull(summary.getProgress());
14+
Assert.assertNotNull(summary.getStatistics());
15+
Assert.assertEquals(summary.getReadBytes(), 0L);
16+
Assert.assertEquals(summary.getReadRows(), 0L);
17+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
18+
Assert.assertEquals(summary.getUpdateCount(), 0L);
19+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
20+
Assert.assertEquals(summary.getWrittenRows(), 0L);
21+
22+
Progress progress = new Progress(1L, 2L, 3L, 4L, 5L);
23+
Statistics stats = new Statistics(6L, 7L, 8L, true, 9L);
24+
summary = new ClickHouseResponseSummary(progress, stats);
25+
Assert.assertTrue(summary.getProgress() == progress);
26+
Assert.assertTrue(summary.getStatistics() == stats);
27+
Assert.assertEquals(summary.getReadBytes(), 2L);
28+
Assert.assertEquals(summary.getReadRows(), 1L);
29+
Assert.assertEquals(summary.getTotalRowsToRead(), 3L);
30+
Assert.assertEquals(summary.getUpdateCount(), 1L);
31+
Assert.assertEquals(summary.getWrittenBytes(), 5L);
32+
Assert.assertEquals(summary.getWrittenRows(), 4L);
33+
}
34+
35+
@Test(groups = { "unit" })
36+
public void testAdd() {
37+
ClickHouseResponseSummary summary = new ClickHouseResponseSummary(null, null);
38+
Assert.assertEquals(summary.getReadBytes(), 0L);
39+
Assert.assertEquals(summary.getReadRows(), 0L);
40+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
41+
Assert.assertEquals(summary.getUpdateCount(), 0L);
42+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
43+
Assert.assertEquals(summary.getWrittenRows(), 0L);
44+
45+
summary.add(summary);
46+
Assert.assertEquals(summary.getReadBytes(), 0L);
47+
Assert.assertEquals(summary.getReadRows(), 0L);
48+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
49+
Assert.assertEquals(summary.getUpdateCount(), 1L);
50+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
51+
Assert.assertEquals(summary.getWrittenRows(), 0L);
52+
53+
summary.add(ClickHouseResponseSummary.EMPTY);
54+
Assert.assertEquals(summary.getReadBytes(), 0L);
55+
Assert.assertEquals(summary.getReadRows(), 0L);
56+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
57+
Assert.assertEquals(summary.getUpdateCount(), 2L);
58+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
59+
Assert.assertEquals(summary.getWrittenRows(), 0L);
60+
61+
summary.add(new Progress(1L, 2L, 3L, 4L, 5L));
62+
Assert.assertEquals(summary.getReadBytes(), 2L);
63+
Assert.assertEquals(summary.getReadRows(), 1L);
64+
Assert.assertEquals(summary.getTotalRowsToRead(), 3L);
65+
Assert.assertEquals(summary.getUpdateCount(), 3L);
66+
Assert.assertEquals(summary.getWrittenBytes(), 5L);
67+
Assert.assertEquals(summary.getWrittenRows(), 4L);
68+
69+
summary.add(new Statistics(6L, 7L, 8L, true, 9L));
70+
Assert.assertEquals(summary.getReadBytes(), 2L);
71+
Assert.assertEquals(summary.getReadRows(), 1L);
72+
Assert.assertEquals(summary.getTotalRowsToRead(), 3L);
73+
Assert.assertEquals(summary.getUpdateCount(), 3L);
74+
Assert.assertEquals(summary.getWrittenBytes(), 5L);
75+
Assert.assertEquals(summary.getWrittenRows(), 4L);
76+
}
77+
78+
@Test(groups = { "unit" })
79+
public void testUpdate() {
80+
ClickHouseResponseSummary summary = new ClickHouseResponseSummary(null, null);
81+
Assert.assertEquals(summary.getReadBytes(), 0L);
82+
Assert.assertEquals(summary.getReadRows(), 0L);
83+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
84+
Assert.assertEquals(summary.getUpdateCount(), 0L);
85+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
86+
Assert.assertEquals(summary.getWrittenRows(), 0L);
87+
88+
summary.update();
89+
Assert.assertEquals(summary.getReadBytes(), 0L);
90+
Assert.assertEquals(summary.getReadRows(), 0L);
91+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
92+
Assert.assertEquals(summary.getUpdateCount(), 1L);
93+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
94+
Assert.assertEquals(summary.getWrittenRows(), 0L);
95+
96+
summary.update(new Progress(1L, 2L, 3L, 4L, 5L));
97+
Assert.assertEquals(summary.getReadBytes(), 2L);
98+
Assert.assertEquals(summary.getReadRows(), 1L);
99+
Assert.assertEquals(summary.getTotalRowsToRead(), 3L);
100+
Assert.assertEquals(summary.getUpdateCount(), 1L);
101+
Assert.assertEquals(summary.getWrittenBytes(), 5L);
102+
Assert.assertEquals(summary.getWrittenRows(), 4L);
103+
104+
summary.update(new Statistics(6L, 7L, 8L, true, 9L));
105+
Assert.assertEquals(summary.getReadBytes(), 2L);
106+
Assert.assertEquals(summary.getReadRows(), 1L);
107+
Assert.assertEquals(summary.getTotalRowsToRead(), 3L);
108+
Assert.assertEquals(summary.getUpdateCount(), 1L);
109+
Assert.assertEquals(summary.getWrittenBytes(), 5L);
110+
Assert.assertEquals(summary.getWrittenRows(), 4L);
111+
}
112+
113+
@Test(groups = { "unit" })
114+
public void testSeal() {
115+
ClickHouseResponseSummary summary = new ClickHouseResponseSummary(null, null);
116+
summary.add(summary);
117+
summary.add(summary.getProgress());
118+
summary.add(summary.getStatistics());
119+
summary.update();
120+
summary.update(summary.getProgress());
121+
summary.update(summary.getStatistics());
122+
Assert.assertEquals(summary.getReadBytes(), 0L);
123+
Assert.assertEquals(summary.getReadRows(), 0L);
124+
Assert.assertEquals(summary.getTotalRowsToRead(), 0L);
125+
Assert.assertEquals(summary.getUpdateCount(), 3L);
126+
Assert.assertEquals(summary.getWrittenBytes(), 0L);
127+
Assert.assertEquals(summary.getWrittenRows(), 0L);
128+
129+
summary.seal();
130+
131+
Assert.assertThrows(IllegalStateException.class, () -> summary.add(summary));
132+
Assert.assertThrows(IllegalStateException.class, () -> summary.add(summary.getProgress()));
133+
Assert.assertThrows(IllegalStateException.class, () -> summary.add(summary.getStatistics()));
134+
135+
Assert.assertThrows(IllegalStateException.class, () -> summary.update());
136+
Assert.assertThrows(IllegalStateException.class, () -> summary.update(summary.getProgress()));
137+
Assert.assertThrows(IllegalStateException.class, () -> summary.update(summary.getStatistics()));
138+
}
139+
}

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseFreezableMap.java

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.clickhouse.data;
22

3+
import java.util.ArrayList;
34
import java.util.Collection;
5+
import java.util.Collections;
6+
import java.util.List;
47
import java.util.Map;
58
import java.util.Set;
69
import java.util.concurrent.atomic.AtomicBoolean;
@@ -10,21 +13,57 @@
1013
* any error.
1114
*/
1215
public class ClickHouseFreezableMap<K, V> implements Map<K, V> {
13-
public static <K, V> ClickHouseFreezableMap<K, V> of(Map<K, V> map) {
14-
return new ClickHouseFreezableMap<>(ClickHouseChecker.nonNull(map, "Map"));
16+
/**
17+
* Creates a freezable map with initial data and optional whitelisted keys.
18+
*
19+
* @param <K> type of the key
20+
* @param <V> type of the value
21+
* @param map non-null map with initial data
22+
* @param whiteListedKeys optional whitelisted keys
23+
* @return non-null freezable map
24+
*/
25+
public static <K, V> ClickHouseFreezableMap<K, V> of(Map<K, V> map, K... whiteListedKeys) {
26+
return new ClickHouseFreezableMap<>(ClickHouseChecker.nonNull(map, "Map"), whiteListedKeys);
1527
}
1628

1729
private final AtomicBoolean freezed;
1830
private final Map<K, V> map;
31+
private final List<K> whitelist;
1932

20-
protected ClickHouseFreezableMap(Map<K, V> map) {
33+
protected ClickHouseFreezableMap(Map<K, V> map, K... keys) {
2134
this.freezed = new AtomicBoolean(false);
2235
this.map = map;
36+
if (keys == null || keys.length == 0) {
37+
this.whitelist = Collections.emptyList();
38+
} else {
39+
List<K> list = new ArrayList<>(keys.length);
40+
for (K k : keys) {
41+
if (k != null && !list.contains(k)) {
42+
list.add(k);
43+
}
44+
}
45+
this.whitelist = Collections.unmodifiableList(list);
46+
}
47+
}
48+
49+
/**
50+
* Checks whether the given key can be used in mutation(e.g. {@code put()},
51+
* {@code remove()}), regardless the map is freezed or not.
52+
*
53+
* @param key non-null key
54+
* @return true if the given key can be used in mutation; false otherwise
55+
*/
56+
protected boolean isMutable(Object key) {
57+
return !freezed.get() || whitelist.contains(key);
2358
}
2459

2560
@Override
2661
public void clear() {
27-
if (!freezed.get()) {
62+
if (freezed.get()) {
63+
for (K k : whitelist) {
64+
map.remove(k);
65+
}
66+
} else {
2867
map.clear();
2968
}
3069
}
@@ -61,19 +100,26 @@ public Set<K> keySet() {
61100

62101
@Override
63102
public V put(K key, V value) {
64-
return !freezed.get() ? map.put(key, value) : value;
103+
return isMutable(key) ? map.put(key, value) : value;
65104
}
66105

67106
@Override
68107
public void putAll(Map<? extends K, ? extends V> m) {
69-
if (!freezed.get()) {
108+
if (freezed.get()) {
109+
for (K k : whitelist) {
110+
V v = m.get(k);
111+
if (v != null) {
112+
map.put(k, v);
113+
}
114+
}
115+
} else {
70116
map.putAll(m);
71117
}
72118
}
73119

74120
@Override
75121
public V remove(Object key) {
76-
return !freezed.get() ? map.remove(key) : null;
122+
return isMutable(key) ? map.remove(key) : null;
77123
}
78124

79125
@Override
@@ -114,4 +160,15 @@ public ClickHouseFreezableMap<K, V> unfreeze() {
114160
public boolean isFreezed() {
115161
return freezed.get();
116162
}
163+
164+
/**
165+
* Checks whether the given key is whitelisted, meaning corresponding entry can
166+
* be changed even the map is freezed.
167+
*
168+
* @param key non-null key
169+
* @return true if the key is whitelisted; false otherwise
170+
*/
171+
public boolean isWhiteListed(K key) {
172+
return key != null && whitelist.contains(key);
173+
}
117174
}

0 commit comments

Comments
 (0)