Skip to content

Commit 93dfb96

Browse files
chernserchernser
authored
Merge pull request #2105 from ClickHouse/v2_fix_random_issue_0122
[client-v2] Small fixes
2 parents fdf6430 + f1a1476 commit 93dfb96

File tree

8 files changed

+124
-16
lines changed

8 files changed

+124
-16
lines changed

‎client-v2/src/main/java/com/clickhouse/client/api/Client.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import com.clickhouse.data.ClickHouseFormat;
4242
import org.apache.hc.core5.concurrent.DefaultThreadFactory;
4343
import org.apache.hc.core5.http.ClassicHttpResponse;
44+
import org.apache.hc.core5.http.Header;
4445
import org.apache.hc.core5.http.HttpHeaders;
4546
import org.apache.hc.core5.http.HttpStatus;
4647
import org.slf4j.Logger;
@@ -1639,8 +1640,13 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
16391640
.getFirstHeader(ClickHouseHttpProto.HEADER_QUERY_ID), finalSettings.getQueryId());
16401641
metrics.setQueryId(queryId);
16411642
metrics.operationComplete();
1643+
Header formatHeader = httpResponse.getFirstHeader(ClickHouseHttpProto.HEADER_FORMAT);
1644+
ClickHouseFormat responseFormat = finalSettings.getFormat();
1645+
if (formatHeader != null) {
1646+
responseFormat = ClickHouseFormat.valueOf(formatHeader.getValue());
1647+
}
16421648

1643-
return new QueryResponse(httpResponse, finalSettings.getFormat(), finalSettings, metrics);
1649+
return new QueryResponse(httpResponse, responseFormat, finalSettings, metrics);
16441650

16451651
} catch (Exception e) {
16461652
lastException = httpClientHelper.wrapException(String.format("Query request failed (Attempt: %s/%s - Duration: %s)",

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,12 @@ public interface ClickHouseBinaryFormatReader extends AutoCloseable {
5151
boolean hasNext();
5252

5353
/**
54-
* Moves cursor to the next row. Must be called before reading the first row.
54+
* Moves cursor to the next row. Must be called before reading the first row. Returns reference to
55+
* an internal record representation. It means that next call to the method will affect value in returned Map.
56+
* This is done for memory usage optimization.
57+
* Method is intended to be used only by the client not an application.
5558
*
56-
* @return map filled with column values or null if no more records are available
59+
* @return reference to a map filled with column values or null if no more records are available
5760
*/
5861
Map<String, Object> next();
5962

‎client-v2/src/main/java/com/clickhouse/client/api/query/QueryResponse.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,15 @@ public long getResultRows() {
133133
return operationMetrics.getMetric(ServerMetrics.RESULT_ROWS).getLong();
134134
}
135135

136+
137+
/**
138+
* Alias for {@link ServerMetrics#TOTAL_ROWS_TO_READ}
139+
* @return estimated number of rows to read
140+
*/
141+
public long getTotalRowsToRead() {
142+
return operationMetrics.getMetric(ServerMetrics.TOTAL_ROWS_TO_READ).getLong();
143+
}
144+
136145
/**
137146
* Alias for {@link OperationMetrics#getQueryId()}
138147
* @return query id of the request

‎client-v2/src/test/java/com/clickhouse/client/insert/SamplePOJO.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ public class SamplePOJO {
8585
private ClickHouseBitmap groupBitmapUint32;
8686
private ClickHouseBitmap groupBitmapUint64;
8787

88+
private String keyword;
89+
8890
public SamplePOJO() {
8991
final Random random = new Random();
9092
byteValue = (byte) random.nextInt();
@@ -180,6 +182,8 @@ public SamplePOJO() {
180182

181183
groupBitmapUint32 = ClickHouseBitmap.wrap(random.ints(5, Integer.MAX_VALUE - 100, Integer.MAX_VALUE).toArray());
182184
groupBitmapUint64 = ClickHouseBitmap.wrap(random.longs(5, Long.MAX_VALUE - 100, Long.MAX_VALUE).toArray());
185+
186+
keyword = "database";
183187
}
184188

185189
public byte getByteValue() {
@@ -574,6 +578,14 @@ public void setGroupBitmapUint64(ClickHouseBitmap groupBitmapUint64) {
574578
this.groupBitmapUint64 = groupBitmapUint64;
575579
}
576580

581+
public String getKeyword() {
582+
return keyword;
583+
}
584+
585+
public void setKeyword(String keyword) {
586+
this.keyword = keyword;
587+
}
588+
577589
@Override
578590
public String toString() {
579591
return "SamplePOJO{" +
@@ -683,7 +695,8 @@ public static String generateTableCreateSQL(String tableName) {
683695
"nested Nested (innerInt Int32, innerString String, " +
684696
"innerNullableInt Nullable(Int32)), " +
685697
"groupBitmapUint32 AggregateFunction(groupBitmap, UInt32), " +
686-
"groupBitmapUint64 AggregateFunction(groupBitmap, UInt64) " +
698+
"groupBitmapUint64 AggregateFunction(groupBitmap, UInt64), " +
699+
"keyword LowCardinality(String) " +
687700
") ENGINE = MergeTree ORDER BY ()";
688701
}
689702
}

‎client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.fasterxml.jackson.databind.JsonNode;
3535
import com.fasterxml.jackson.databind.MappingIterator;
3636
import com.fasterxml.jackson.databind.ObjectMapper;
37+
import com.google.common.io.BaseEncoding;
3738
import org.apache.commons.lang3.RandomStringUtils;
3839
import org.apache.commons.lang3.StringEscapeUtils;
3940
import org.testng.Assert;
@@ -55,13 +56,15 @@
5556
import java.net.Inet4Address;
5657
import java.net.Inet6Address;
5758
import java.net.InetAddress;
59+
import java.nio.ByteBuffer;
5860
import java.nio.charset.StandardCharsets;
5961
import java.time.LocalDate;
6062
import java.time.LocalDateTime;
6163
import java.time.ZoneId;
6264
import java.time.ZonedDateTime;
6365
import java.util.ArrayList;
6466
import java.util.Arrays;
67+
import java.util.Collection;
6568
import java.util.Collections;
6669
import java.util.HashMap;
6770
import java.util.HashSet;
@@ -486,7 +489,8 @@ record = reader.next();
486489
"col1 Array(UInt32)",
487490
"col2 Array(Array(Int32))",
488491
"col3 Array(UInt64)",
489-
"col4 Array(Bool)"
492+
"col4 Array(Bool)",
493+
"col5 Array(String)"
490494
);
491495

492496
private final static List<Function<String, Object>> ARRAY_VALUE_GENERATORS = Arrays.asList(
@@ -505,8 +509,17 @@ record = reader.next();
505509
RANDOM.longs(10, 0, Long.MAX_VALUE)
506510
.mapToObj(BigInteger::valueOf).collect(Collectors.toList()),
507511
c -> RANDOM.ints(10, 0, 1)
508-
.mapToObj(i -> i == 0 ).collect(Collectors.toList())
509-
512+
.mapToObj(i -> i == 0 ).collect(Collectors.toList()),
513+
c -> {
514+
UUID uuid = UUID.randomUUID();
515+
byte[] bts = ByteBuffer.allocate(16)
516+
.putLong(uuid.getMostSignificantBits())
517+
.putLong(uuid.getLeastSignificantBits())
518+
.array();
519+
String sep = "\\x";
520+
String hex = sep + BaseEncoding.base16().withSeparator(sep, 2).encode(bts);
521+
return Arrays.asList(hex);
522+
}
510523
);
511524

512525
@Test(groups = {"integration"})
@@ -546,6 +559,7 @@ public void testArrayValues() throws Exception {
546559
Assert.assertEquals(col4Values, data.get(0).get("col4"));
547560
boolean[] col4Array = reader.getBooleanArray("col4");
548561
Assert.assertEquals(col4Array, ((List)data.get(0).get("col4")).toArray());
562+
Assert.assertEquals(reader.getList("col5"), ((List)data.get(0).get("col5")));
549563
}
550564

551565
@Test
@@ -1452,6 +1466,13 @@ private Map<String, Object> writeValuesRow(StringBuilder insertStmtBuilder, List
14521466
}
14531467
insertStmtBuilder.setLength(insertStmtBuilder.length() - 2);
14541468
insertStmtBuilder.append("}, ");
1469+
} else if (value instanceof List) {
1470+
insertStmtBuilder.append("[");
1471+
for (Object item : (List)value) {
1472+
insertStmtBuilder.append(quoteValue(item)).append(", ");
1473+
}
1474+
insertStmtBuilder.setLength(insertStmtBuilder.length() - 2);
1475+
insertStmtBuilder.append("], ");
14551476
} else {
14561477
insertStmtBuilder.append(value).append(", ");
14571478
}
@@ -1463,7 +1484,9 @@ private Map<String, Object> writeValuesRow(StringBuilder insertStmtBuilder, List
14631484

14641485
private String quoteValue(Object value) {
14651486
if (value instanceof String) {
1466-
return '\'' + value.toString() + '\'';
1487+
String strVal = (String)value;
1488+
1489+
return '\'' + strVal.replaceAll("\\\\", "\\\\\\\\") + '\'';
14671490
}
14681491
return value.toString();
14691492
}
@@ -2000,4 +2023,37 @@ public void testServerTimezone() throws Exception {
20002023
Assert.assertEquals(serverUtcTime.withZoneSameInstant(ZoneId.of("America/New_York")), serverEstTime);
20012024
}
20022025
}
2026+
2027+
@Test(groups = {"integration"})
2028+
public void testLowCardinalityValues() throws Exception {
2029+
final String table = "test_low_cardinality_values";
2030+
final String tableCreate = "CREATE TABLE " + table + "(rowID Int32, keyword LowCardinality(String)) Engine = MergeTree ORDER BY ()";
2031+
2032+
client.execute("DROP TABLE IF EXISTS " + table);
2033+
client.execute(tableCreate);
2034+
2035+
client.execute("INSERT INTO " + table + " VALUES (0, 'db'), (1, 'fast'), (2, 'not a java')");
2036+
String[] values = new String[] {"db", "fast", "not a java"};
2037+
Collection<GenericRecord> records = client.queryAll("SELECT * FROM " + table);
2038+
for (GenericRecord record : records) {
2039+
int rowId = record.getInteger("rowID");
2040+
Assert.assertEquals(record.getString("keyword"), values[rowId]);
2041+
}
2042+
}
2043+
2044+
@Test(groups = {"integration"})
2045+
public void testGettingRowsBeforeLimit() throws Exception {
2046+
int expectedTotalRowsToRead = 100;
2047+
List<GenericRecord> serverVersion = client.queryAll("SELECT version()");
2048+
if (ClickHouseVersion.of(serverVersion.get(0).getString(1)).check("(,23.8]")) {
2049+
// issue in prev. release.
2050+
expectedTotalRowsToRead = 0;
2051+
}
2052+
2053+
try (QueryResponse response = client.query("SELECT number FROM system.numbers LIMIT 100").get()) {
2054+
Assert.assertTrue(response.getResultRows() < 1000);
2055+
2056+
Assert.assertEquals(response.getTotalRowsToRead(), expectedTotalRowsToRead);
2057+
}
2058+
}
20032059
}

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,11 @@ public ResultSetImpl executeQuery(String sql, QuerySettings settings) throws SQL
182182
} else {
183183
response = connection.client.query(lastSql, mergedSettings).get(queryTimeout, TimeUnit.SECONDS);
184184
}
185+
186+
if (response.getFormat().isText()) {
187+
throw new SQLException("Only RowBinaryWithNameAndTypes is supported for output format. Please check your query.",
188+
ExceptionUtils.SQL_STATE_CLIENT_ERROR);
189+
}
185190
ClickHouseBinaryFormatReader reader = connection.client.newBinaryFormatReader(response);
186191

187192
currentResultSet = new ResultSetImpl(this, response, reader);
@@ -225,7 +230,7 @@ public int executeUpdate(String sql, QuerySettings settings) throws SQLException
225230
try (QueryResponse response = queryTimeout == 0 ? connection.client.query(lastSql, mergedSettings).get()
226231
: connection.client.query(lastSql, mergedSettings).get(queryTimeout, TimeUnit.SECONDS)) {
227232
currentResultSet = null;
228-
updateCount = (int) response.getWrittenRows();
233+
updateCount = Math.max(0, (int) response.getWrittenRows()); // when statement alters schema no result rows returned.
229234
metrics = response.getMetrics();
230235
lastQueryId = response.getQueryId();
231236
} catch (Exception e) {
@@ -601,4 +606,12 @@ public String enquoteNCharLiteral(String val) throws SQLException {
601606
checkClosed();
602607
return Statement.super.enquoteNCharLiteral(val);
603608
}
609+
610+
/**
611+
* Return query ID of last executed statement. It is not guaranteed when statements is used concurrently.
612+
* @return query ID
613+
*/
614+
public String getLastQueryId() {
615+
return lastQueryId;
616+
}
604617
}

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package com.clickhouse.jdbc;
22

3-
import com.clickhouse.client.api.ClientConfigProperties;
43
import com.clickhouse.data.Tuple;
54
import org.slf4j.Logger;
65
import org.slf4j.LoggerFactory;
@@ -11,7 +10,6 @@
1110
import java.math.BigInteger;
1211
import java.sql.Connection;
1312
import java.sql.Date;
14-
import java.sql.DriverManager;
1513
import java.sql.JDBCType;
1614
import java.sql.PreparedStatement;
1715
import java.sql.ResultSet;
@@ -26,7 +24,6 @@
2624
import java.util.GregorianCalendar;
2725
import java.util.HashMap;
2826
import java.util.Map;
29-
import java.util.Properties;
3027
import java.util.Random;
3128
import java.util.TimeZone;
3229
import java.util.UUID;
@@ -318,7 +315,8 @@ public void testStringTypes() throws SQLException {
318315
runQuery("CREATE TABLE test_strings (order Int8, "
319316
+ "str String, fixed FixedString(6), "
320317
+ "enum Enum8('a' = 6, 'b' = 7, 'c' = 8), enum8 Enum8('a' = 1, 'b' = 2, 'c' = 3), enum16 Enum16('a' = 1, 'b' = 2, 'c' = 3), "
321-
+ "uuid UUID, ipv4 IPv4, ipv6 IPv6"
318+
+ "uuid UUID, ipv4 IPv4, ipv6 IPv6, "
319+
+ "escaped String "
322320
+ ") ENGINE = MergeTree ORDER BY ()");
323321

324322
// Insert random (valid) values
@@ -333,10 +331,10 @@ public void testStringTypes() throws SQLException {
333331
String uuid = UUID.randomUUID().toString();
334332
String ipv4 = rand.nextInt(256) + "." + rand.nextInt(256) + "." + rand.nextInt(256) + "." + rand.nextInt(256);
335333
String ipv6 = "2001:adb8:85a3:1:2:8a2e:370:7334";
336-
334+
String escaped = "\\xA3\\xA3\\x12\\xA0\\xDF\\x13\\x4E\\x8C\\x87\\x74\\xD4\\x53\\xDB\\xFC\\x34\\x95";
337335

338336
try (Connection conn = getConnection()) {
339-
try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_strings VALUES ( 1, ?, ?, ?, ?, ?, ?, ?, ? )")) {
337+
try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO test_strings VALUES ( 1, ?, ?, ?, ?, ?, ?, ?, ?, ? )")) {
340338
stmt.setString(1, str);
341339
stmt.setString(2, fixed);
342340
stmt.setString(3, enum8);
@@ -345,6 +343,7 @@ public void testStringTypes() throws SQLException {
345343
stmt.setString(6, uuid);
346344
stmt.setString(7, ipv4);
347345
stmt.setString(8, ipv6);
346+
stmt.setString(9, escaped);
348347
stmt.executeUpdate();
349348
}
350349
}
@@ -365,7 +364,7 @@ public void testStringTypes() throws SQLException {
365364
assertEquals(rs.getString("uuid"), uuid);
366365
assertEquals(rs.getString("ipv4"), "/" + ipv4);
367366
assertEquals(rs.getString("ipv6"), "/" + ipv6);
368-
367+
assertEquals(rs.getString("escaped"), escaped);
369368
assertFalse(rs.next());
370369
}
371370
}

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ public void testExecuteQuerySimpleNumbers() throws Exception {
5757
assertEquals(rs.getLong("num"), 1);
5858
assertFalse(rs.next());
5959
}
60+
Assert.assertFalse(((StatementImpl)stmt).getLastQueryId().isEmpty());
6061
}
6162
}
6263
}
@@ -530,4 +531,12 @@ public void testConcurrentCancel() throws Exception {
530531
}
531532
}
532533
}
534+
535+
@Test(groups = {"integration"})
536+
public void testTextFormatInResponse() throws Exception {
537+
try (Connection conn = getJdbcConnection();
538+
Statement stmt = conn.createStatement()) {
539+
Assert.expectThrows(SQLException.class, () ->stmt.executeQuery("SELECT 1 FORMAT JSON"));
540+
}
541+
}
533542
}

0 commit comments

Comments
 (0)