Merge pull request #855 from zhicwu/prepare-patch

Prepare patch6

Author: zhicwu

README.md

@@ -112,7 +112,7 @@ Note: in general, the new driver(v0.3.2) is a few times faster with less memory
     <groupId>com.clickhouse</groupId>
     <!-- or clickhouse-grpc-client if you prefer gRPC -->
     <artifactId>clickhouse-http-client</artifactId>
-    <version>0.3.2-patch5</version>
+    <version>0.3.2-patch6</version>
 </dependency>
 ```
 
@@ -148,7 +148,7 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol);
     <!-- will stop using ru.yandex.clickhouse starting from 0.4.0  -->
     <groupId>com.clickhouse</groupId>
     <artifactId>clickhouse-jdbc</artifactId>
-    <version>0.3.2-patch5</version>
+    <version>0.3.2-patch6</version>
     <!-- below is only needed when all you want is a shaded jar -->
     <classifier>http</classifier>
     <exclusions>

clickhouse-client/README.md

@@ -9,7 +9,7 @@ Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so
 <dependency>
     <groupId>com.clickhouse</groupId>
     <artifactId>clickhouse-http-client</artifactId>
-    <version>0.3.2-patch5</version>
+    <version>0.3.2-patch6</version>
 </dependency>
 ```
 

clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseOffsetDateTimeValue.java

@@ -356,7 +356,8 @@ public ClickHouseOffsetDateTimeValue update(String value) {
         if (value == null) {
             resetToNullOrEmpty();
         } else {
-            set(OffsetDateTime.parse(value, ClickHouseValues.DATETIME_FORMATTER));
+            set(LocalDateTime.parse(value, ClickHouseValues.DATETIME_FORMATTER).atZone(tz.toZoneId())
+                    .toOffsetDateTime());
         }
         return this;
     }

clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java

@@ -286,7 +286,7 @@ private void buildMappingsForDataTypes() {
             buildMappings(deserializers, serializers,
                     (r, f, c, i) -> ClickHouseDateValue.of(r,
                             BinaryStreamUtils.readDate32(i, f.getTimeZoneForDate())),
-                    (v, f, c, o) -> BinaryStreamUtils.writeDate(o, v.asDate(), f.getTimeZoneForDate()),
+                    (v, f, c, o) -> BinaryStreamUtils.writeDate32(o, v.asDate(), f.getTimeZoneForDate()),
                     ClickHouseDataType.Date32);
             buildMappings(deserializers, serializers, (r, f, c, i) -> c.getTimeZone() == null
                     ? ClickHouseDateTimeValue.of(r,

clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java

@@ -414,7 +414,7 @@ public ClickHouseByteArrayValue update(String value) {
                 byte[] arr = new byte[list.size()];
                 int index = 0;
                 for (String v : list) {
-                    arr[index++] = Byte.parseByte(v);
+                    arr[index++] = v == null ? (byte) 0 : Byte.parseByte(v);
                 }
                 set(arr);
             }

clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java

@@ -410,7 +410,7 @@ public ClickHouseDoubleArrayValue update(String value) {
                 double[] arr = new double[list.size()];
                 int index = 0;
                 for (String v : list) {
-                    arr[index++] = Double.parseDouble(v);
+                    arr[index++] = v == null ? 0D : Double.parseDouble(v);
                 }
                 set(arr);
             }

clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java

@@ -410,7 +410,7 @@ public ClickHouseFloatArrayValue update(String value) {
                 float[] arr = new float[list.size()];
                 int index = 0;
                 for (String v : list) {
-                    arr[index++] = Float.parseFloat(v);
+                    arr[index++] = v == null ? 0F : Float.parseFloat(v);
                 }
                 set(arr);
             }

clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java

@@ -410,7 +410,7 @@ public ClickHouseIntArrayValue update(String value) {
                 int[] arr = new int[list.size()];
                 int index = 0;
                 for (String v : list) {
-                    arr[index++] = Integer.parseInt(v);
+                    arr[index++] = v == null ? 0 : Integer.parseInt(v);
                 }
                 set(arr);
             }

clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java

@@ -410,7 +410,7 @@ public ClickHouseLongArrayValue update(String value) {
                 long[] arr = new long[list.size()];
                 int index = 0;
                 for (String v : list) {
-                    arr[index++] = Long.parseLong(v);
+                    arr[index++] = v == null ? 0L : Long.parseLong(v);
                 }
                 set(arr);
             }

clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java

@@ -410,7 +410,7 @@ public ClickHouseShortArrayValue update(String value) {
                 short[] arr = new short[list.size()];
                 int index = 0;
                 for (String v : list) {
-                    arr[index++] = Short.parseShort(v);
+                    arr[index++] = v == null ? (short) 0 : Short.parseShort(v);
                 }
                 set(arr);
             }

clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java

@@ -375,6 +375,11 @@ public void testReadValueArray() {
                 args.lastIndexOf(']') + 1);
         Assert.assertEquals(list, Arrays.asList("1", "2", "3"));
 
+        args = "[1, null , 3,, Null]";
+        list.clear();
+        Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.length());
+        Assert.assertEquals(list, Arrays.asList("1", null, "3", null, null));
+
         args = "['1\\'2', '2,3' , '3\n4\r5']";
         list.clear();
         Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.length());

clickhouse-jdbc/README.md

@@ -11,7 +11,7 @@ Keep in mind that `clickhouse-jdbc` is synchronous, and in general it has more o
     <!-- will stop using ru.yandex.clickhouse starting from 0.4.0 -->
     <groupId>com.clickhouse</groupId>
     <artifactId>clickhouse-jdbc</artifactId>
-    <version>0.3.2-patch5</version>
+    <version>0.3.2-patch6</version>
 </dependency>
 ```
 
@@ -48,6 +48,10 @@ Note: please refer to [JDBC specific configuration](https://github.com/ClickHous
 
 ```java
 String url = "jdbc:ch://my-server/system"; // use http protocol and port 8123 by default
+// String url = "jdbc:ch://my-server:8443/system"; // if you prefer https
+Properties properties = new Properties();
+// properties.setProperty("ssl", "true");
+// properties.setProperty("sslmode", "NONE"); // NONE to trust all servers; STRICT for trusted only
 ClickHouseDataSource dataSource = new ClickHouseDataSource(url, new Properties());
 try (Connection conn = dataSource.getConnection("default", "password");
     Statement stmt = conn.createStatement()) {
@@ -69,7 +73,7 @@ Tips:
 2. Use [input function](https://clickhouse.com/docs/en/sql-reference/table-functions/input/) whenever possible
 
 ```java
-// create table mytable(id String, ts DateTime64(3), desc Nullable(String)) engine=Memory
+// create table mytable(id String, timestamp DateTime64(3), description Nullable(String)) engine=Memory
 
 // 1. recommended as it performs the best
 try (PreparedStatement ps = conn.prepareStatement(
@@ -78,27 +82,28 @@ try (PreparedStatement ps = conn.prepareStatement(
     ps.setString(1, "test"); // col1
     ps.setObject(2, LocalDateTime.now()); // col2, setTimestamp is slow and not recommended
     ps.setInt(3, 123); // col3
-    ps.addBatch(); // parameters will be write into stream in binary format
+    ps.addBatch(); // parameters will be write into buffered stream immediately in binary format
     ...
     ps.executeBatch(); // stream everything on-hand into ClickHouse
 }
 
 // 2. easier to use but slower compare to input function
 try (PreparedStatement ps = conn.prepareStatement("insert into mytable(* except (desc))")) {
-    // the driver will issue query "select * except (desc) from mytable where 0" for type inferring
-    // since desc column is excluded, we know there are only two parameters: col1 and col2
-    ps.setString(1, "test"); // col1
-    ps.setObject(2, LocalDateTime.now()); // col2
-    ps.addBatch(); // parameters will be write into stream in binary format
+    // the driver will issue query "select * except (description) from mytable where 0" for type inferring
+    // since description column is excluded, we know there are only two parameters: col1 and col2
+    ps.setString(1, "test"); // id
+    ps.setObject(2, LocalDateTime.now()); // timestamp
+    ps.addBatch(); // parameters will be write into buffered stream immediately in binary format
     ...
     ps.executeBatch(); // stream everything on-hand into ClickHouse
 }
 
 // 3. not recommended as it's based on a large SQL
-try (PreparedStatement ps = conn.prepareStatement("insert into mytable values(?,?,?)")) {
-    ps.setString(1, "test"); // col1
-    ps.setObject(2, LocalDateTime.now()); // col2
-    ps.setString(3, null); // col3
+// Note: "insert into mytable values(?,?,?)" is treated as "insert into mytable"
+try (PreparedStatement ps = conn.prepareStatement("insert into mytable values(trim(?),?,?)")) {
+    ps.setString(1, "test"); // id
+    ps.setObject(2, LocalDateTime.now()); // timestamp
+    ps.setString(3, null); // description
     ps.addBatch(); // append parameters to the query
     ...
     ps.executeBatch(); // issue the composed query: insert into mytable values(...)(...)...(...)

clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/AbstractPreparedStatement.java

@@ -5,6 +5,7 @@
 import java.sql.SQLException;
 
 import com.clickhouse.client.ClickHouseRequest;
+import com.clickhouse.client.ClickHouseUtils;
 import com.clickhouse.jdbc.SqlExceptionUtils;
 
 public abstract class AbstractPreparedStatement extends ClickHouseStatementImpl implements PreparedStatement {
@@ -15,6 +16,25 @@ protected AbstractPreparedStatement(ClickHouseConnectionImpl connection, ClickHo
 
     protected abstract long[] executeAny(boolean asBatch) throws SQLException;
 
+    protected abstract int getMaxParameterIndex();
+
+    protected int toArrayIndex(int parameterIndex) throws SQLException {
+        int max = getMaxParameterIndex();
+        if (max < 1) {
+            String name = getConnection().getJdbcConfig().useNamedParameter() ? "named parameter"
+                    : "JDBC style '?' placeholder";
+            throw SqlExceptionUtils.clientError(ClickHouseUtils
+                    .format("Can't set parameter at index %d due to no %s found in the query",
+                            parameterIndex, name));
+        } else if (parameterIndex < 1 || parameterIndex > max) {
+            throw SqlExceptionUtils.clientError(ClickHouseUtils
+                    .format("Parameter index must between 1 and %d but we got %d", max,
+                            parameterIndex));
+        }
+
+        return parameterIndex - 1;
+    }
+
     @Override
     public final void addBatch(String sql) throws SQLException {
         ensureOpen();

clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java

@@ -594,10 +594,35 @@ public PreparedStatement prepareStatement(String sql, int resultSetType, int res
                 throw SqlExceptionUtils
                         .clientError(
                                 "External table, input function, and query parameter cannot be used together in PreparedStatement.");
+            } else if (parsedStmt.getStatementType() == StatementType.INSERT &&
+                    !parsedStmt.containsKeyword("SELECT") && parsedStmt.hasValues() &&
+                    (!parsedStmt.hasFormat() || clientRequest.getFormat().name().equals(parsedStmt.getFormat()))) {
+                String query = parsedStmt.getSQL();
+                int startIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_START);
+                int endIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_END);
+                boolean useStream = true;
+                for (int i = startIndex + 1; i < endIndex; i++) {
+                    char ch = query.charAt(i);
+                    if (ch != '?' && ch != ',' && !Character.isWhitespace(ch)) {
+                        useStream = false;
+                        break;
+                    }
+                }
+
+                if (useStream) {
+                    ps = new InputBasedPreparedStatement(this,
+                            clientRequest.write().query(query.substring(0, parsedStmt.getStartPosition("VALUES")),
+                                    newQueryId()),
+                            getTableColumns(parsedStmt.getDatabase(), parsedStmt.getTable(),
+                                    parsedStmt.getContentBetweenKeywords(
+                                            ClickHouseSqlStatement.KEYWORD_TABLE_COLUMNS_START,
+                                            ClickHouseSqlStatement.KEYWORD_TABLE_COLUMNS_END)),
+                            resultSetType, resultSetConcurrency, resultSetHoldability);
+                }
             }
         } else {
             if (parsedStmt.hasTempTable()) {
-                // non-insert queries using temp table
+                // queries using external/temporary table
                 ps = new TableBasedPreparedStatement(this,
                         clientRequest.write().query(parsedStmt.getSQL(), newQueryId()), parsedStmt,
                         resultSetType, resultSetConcurrency, resultSetHoldability);

clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java

@@ -137,13 +137,9 @@ protected long[] executeAny(boolean asBatch) throws SQLException {
         return results;
     }
 
-    protected int toArrayIndex(int parameterIndex) throws SQLException {
-        if (parameterIndex < 1 || parameterIndex > values.length) {
-            throw SqlExceptionUtils.clientError(ClickHouseUtils
-                    .format("Parameter index must between 1 and %d but we got %d", values.length, parameterIndex));
-        }
-
-        return parameterIndex - 1;
+    @Override
+    protected int getMaxParameterIndex() {
+        return values.length;
     }
 
     @Override

clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/SqlBasedPreparedStatement.java

@@ -209,13 +209,9 @@ protected long[] executeAny(boolean asBatch) throws SQLException {
         return results;
     }
 
-    protected int toArrayIndex(int parameterIndex) throws SQLException {
-        if (parameterIndex < 1 || parameterIndex > templates.length) {
-            throw SqlExceptionUtils.clientError(ClickHouseUtils
-                    .format("Parameter index must between 1 and %d but we got %d", templates.length, parameterIndex));
-        }
-
-        return parameterIndex - 1;
+    @Override
+    protected int getMaxParameterIndex() {
+        return templates.length;
     }
 
     @Override

clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/TableBasedPreparedStatement.java

@@ -113,21 +113,17 @@ public long[] executeAny(boolean asBatch) throws SQLException {
         return results;
     }
 
+    @Override
+    protected int getMaxParameterIndex() {
+        return values.length;
+    }
+
     protected String getSql() {
         // why? because request can be modified so it might not always same as
         // parsedStmt.getSQL()
         return getRequest().getStatements(false).get(0);
     }
 
-    protected int toArrayIndex(int parameterIndex) throws SQLException {
-        if (parameterIndex < 1 || parameterIndex > values.length) {
-            throw SqlExceptionUtils.clientError(ClickHouseUtils
-                    .format("Parameter index must between 1 and %d but we got %d", values.length, parameterIndex));
-        }
-
-        return parameterIndex - 1;
-    }
-
     @Override
     public ResultSet executeQuery() throws SQLException {
         ensureParams();