Merge pull request #1343 from zhicwu/main

Add object mapping

Author: zhicwu

CHANGELOG.md

@@ -1,6 +1,8 @@
 ## 0.4.6, 2023-04-26
 ### New Features
 * ClickHouseStatement.setMirroredOutput() for dumping ResultSet.
+* ClickHouseResponse.records(Class<?>) for object mapping.
+* Two new options(use_compilation & max_mapper_cache) reserved for future usage.
 
 ### Bug Fixes
 * NoClassDefFoundError with clickhouse-apache-http-client-jdbc. [#1319](https://github.com/ClickHouse/clickhouse-java/issues/1319)

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java

@@ -204,6 +204,7 @@ public static Map<ClickHouseOption, Serializable> toClientOptions(Map<?, ?> prop
     private final ClickHouseBufferingMode requestBuffering;
     private final ClickHouseBufferingMode responseBuffering;
     private final int maxExecutionTime;
+    private final int maxMapperCache;
     private final int maxQueuedBuffers;
     private final int maxQueuedRequests;
     private final long maxResultRows;
@@ -229,6 +230,7 @@ public static Map<ClickHouseOption, Serializable> toClientOptions(Map<?, ?> prop
     private final boolean widenUnsignedTypes;
     private final boolean useBinaryString;
     private final boolean useBlockingQueue;
+    private final boolean useCompilation;
     private final boolean useObjectsInArray;
     private final boolean useNoProxy;
     private final boolean useServerTimeZone;
@@ -312,6 +314,7 @@ public ClickHouseConfig(Map<ClickHouseOption, Serializable> options, ClickHouseC
         this.responseBuffering = (ClickHouseBufferingMode) getOption(ClickHouseClientOption.RESPONSE_BUFFERING,
                 ClickHouseDefaults.BUFFERING);
         this.maxExecutionTime = getIntOption(ClickHouseClientOption.MAX_EXECUTION_TIME);
+        this.maxMapperCache = getIntOption(ClickHouseClientOption.MAX_MAPPER_CACHE);
         this.maxQueuedBuffers = getIntOption(ClickHouseClientOption.MAX_QUEUED_BUFFERS);
         this.maxQueuedRequests = getIntOption(ClickHouseClientOption.MAX_QUEUED_REQUESTS);
         this.maxResultRows = getLongOption(ClickHouseClientOption.MAX_RESULT_ROWS);
@@ -340,6 +343,7 @@ public ClickHouseConfig(Map<ClickHouseOption, Serializable> options, ClickHouseC
         this.widenUnsignedTypes = getBoolOption(ClickHouseClientOption.WIDEN_UNSIGNED_TYPES);
         this.useBinaryString = getBoolOption(ClickHouseClientOption.USE_BINARY_STRING);
         this.useBlockingQueue = getBoolOption(ClickHouseClientOption.USE_BLOCKING_QUEUE);
+        this.useCompilation = getBoolOption(ClickHouseClientOption.USE_COMPILATION);
         this.useObjectsInArray = getBoolOption(ClickHouseClientOption.USE_OBJECTS_IN_ARRAYS);
         this.useNoProxy = getBoolOption(ClickHouseClientOption.USE_NO_PROXY);
         this.useServerTimeZone = getBoolOption(ClickHouseClientOption.USE_SERVER_TIME_ZONE);
@@ -458,6 +462,11 @@ public int getMaxBufferSize() {
         return maxBufferSize;
     }
 
+    @Override
+    public int getMaxMapperCache() {
+        return maxMapperCache;
+    }
+
     @Override
     public int getBufferSize() {
         return bufferSize;
@@ -615,6 +624,11 @@ public boolean isUseBlockingQueue() {
         return useBlockingQueue;
     }
 
+    @Override
+    public boolean isUseCompilation() {
+        return useCompilation;
+    }
+
     @Override
     public boolean isUseObjectsInArray() {
         return useObjectsInArray;

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseParameterizedQuery.java

@@ -548,11 +548,11 @@ public List<String[]> getQueryParts() {
      */
     public ClickHouseValue[] getParameterTemplates() {
         int i = 0;
-        ClickHouseValue[] tempaltes = new ClickHouseValue[names.size()];
+        ClickHouseValue[] templates = new ClickHouseValue[names.size()];
         for (ClickHouseValue v : names.values()) {
-            tempaltes[i++] = v;
+            templates[i++] = v;
         }
-        return tempaltes;
+        return templates;
     }
 
     /**

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseResponse.java

@@ -52,14 +52,19 @@ public ClickHouseResponseSummary getSummary() {
 
         @Override
         public ClickHouseInputStream getInputStream() {
-            return null;
+            return ClickHouseInputStream.empty();
         }
 
         @Override
         public Iterable<ClickHouseRecord> records() {
             return Collections.emptyList();
         }
 
+        @Override
+        public <T> Iterable<T> records(Class<T> objClass) {
+            return Collections.emptyList();
+        }
+
         @Override
         public void close() {
             // do nothing
@@ -93,7 +98,7 @@ public boolean isClosed() {
      * also means additional work is required for deserialization, especially when
      * using a binary format.
      *
-     * @return input stream for getting raw data returned from server
+     * @return non-null input stream for getting raw data returned from server
      */
     ClickHouseInputStream getInputStream();
 
@@ -109,6 +114,20 @@ default ClickHouseRecord firstRecord() {
         return records().iterator().next();
     }
 
+    /**
+     * Gets the first record as mapped object. Please use {@link #records(Class)}
+     * instead if you need to access the rest of records.
+     * 
+     * @param <T>      type of the mapped object
+     * @param objClass non-null class of the mapped object
+     * @return mapped object of the first record
+     * @throws NoSuchElementException when there's no record at all
+     * @throws UncheckedIOException   when failed to read data(e.g. deserialization)
+     */
+    default <T> T firstRecord(Class<T> objClass) {
+        return records(objClass).iterator().next();
+    }
+
     /**
      * Returns an iterable collection of records which can be walked through in a
      * foreach loop. Please pay attention that: 1) {@link UncheckedIOException}
@@ -120,6 +139,18 @@ default ClickHouseRecord firstRecord() {
      */
     Iterable<ClickHouseRecord> records();
 
+    /**
+     * Returns an iterable collection of mapped objects which can be walked through
+     * in a foreach loop. When {@code objClass} is null or {@link ClickHouseRecord},
+     * it's same as calling {@link #records()}.
+     *
+     * @param <T>      type of the mapped object
+     * @param objClass non-null class of the mapped object
+     * @return non-null iterable collection
+     * @throws UncheckedIOException when failed to read data(e.g. deserialization)
+     */
+    <T> Iterable<T> records(Class<T> objClass);
+
     /**
      * Pipes the contents of this response into the given output stream. Keep in
      * mind that it's caller's responsibility to flush and close the output stream.
@@ -146,6 +177,16 @@ default Stream<ClickHouseRecord> stream() {
                 Spliterator.IMMUTABLE | Spliterator.NONNULL | Spliterator.ORDERED), false);
     }
 
+    /**
+     * Gets stream of mapped objects to process.
+     *
+     * @return stream of mapped objects
+     */
+    default <T> Stream<T> stream(Class<T> objClass) {
+        return StreamSupport.stream(Spliterators.spliteratorUnknownSize(records(objClass).iterator(),
+                Spliterator.IMMUTABLE | Spliterator.NONNULL | Spliterator.ORDERED), false);
+    }
+
     @Override
     void close();
 

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseSimpleResponse.java

@@ -8,6 +8,7 @@
 import com.clickhouse.data.ClickHouseColumn;
 import com.clickhouse.data.ClickHouseInputStream;
 import com.clickhouse.data.ClickHouseRecord;
+import com.clickhouse.data.ClickHouseRecordMapper;
 import com.clickhouse.data.ClickHouseRecordTransformer;
 import com.clickhouse.data.ClickHouseSimpleRecord;
 import com.clickhouse.data.ClickHouseValue;
@@ -169,6 +170,16 @@ public Iterable<ClickHouseRecord> records() {
         return records;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> Iterable<T> records(Class<T> objClass) {
+        if (objClass == null || objClass == ClickHouseRecord.class) {
+            return (Iterable<T>) records();
+        }
+
+        return () -> ClickHouseRecordMapper.wrap(null, columns, records().iterator(), objClass, null);
+    }
+
     @Override
     public void close() {
         // nothing to close

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseStreamResponse.java

@@ -141,7 +141,15 @@ public Iterable<ClickHouseRecord> records() {
             throw new UnsupportedOperationException(
                     "No data processor available for deserialization, please consider to use getInputStream instead");
         }
-
         return processor.records();
     }
+
+    @Override
+    public <T> Iterable<T> records(Class<T> objClass) {
+        if (processor == null) {
+            throw new UnsupportedOperationException(
+                    "No data processor available for deserialization, please consider to use getInputStream instead");
+        }
+        return processor.records(objClass, null);
+    }
 }

clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java

@@ -182,6 +182,11 @@ public enum ClickHouseClientOption implements ClickHouseOption {
      */
     MAX_BUFFER_SIZE("max_buffer_size", ClickHouseDataConfig.DEFAULT_MAX_BUFFER_SIZE,
             "Maximum buffer size in byte can be used for streaming."),
+    /**
+     * Maximum number of mappers can be cached.
+     */
+    MAX_MAPPER_CACHE("max_mapper_cache", ClickHouseDataConfig.DEFAULT_MAX_MAPPER_CACHE,
+            "Maximum number of mappers can be cached."),
     /**
      * Maximum query execution time in seconds.
      */
@@ -350,6 +355,12 @@ public enum ClickHouseClientOption implements ClickHouseOption {
      */
     USE_BLOCKING_QUEUE("use_blocking_queue", ClickHouseDataConfig.DEFAULT_USE_BLOCKING_QUEUE,
             "Whether to use blocking queue for buffering."),
+    /**
+     * Whether to use compilation(generated byte code) in object mapping and
+     * serialization.
+     */
+    USE_COMPILATION("use_compilation", ClickHouseDataConfig.DEFAULT_USE_COMPILATION,
+            "Whether to use compilation(generated byte code) in object mapping and serialization."),
     /**
      * Whether Object[] should be used instead of primitive arrays.
      */

clickhouse-data/pom.xml

@@ -85,6 +85,11 @@
             <artifactId>msgpack-core</artifactId>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>org.ow2.asm</groupId>
+            <artifactId>asm</artifactId>
+            <optional>true</optional>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseByteBuffer.java

@@ -1,11 +1,14 @@
 package com.clickhouse.data;
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
 import java.util.Arrays;
 import java.util.List;
 import java.util.UUID;
@@ -144,6 +147,14 @@ public BigInteger asBigInteger() {
         return getBigInteger(0, length, false);
     }
 
+    public BigDecimal asBigDecimal() {
+        return asBigDecimal(0);
+    }
+
+    public BigDecimal asBigDecimal(int scale) {
+        return new BigDecimal(asBigInteger(), scale);
+    }
+
     public boolean asBoolean() {
         return getBoolean(0);
     }
@@ -171,6 +182,19 @@ public double[] asDoubleArray() {
         return values;
     }
 
+    public LocalDate asDate() {
+        return LocalDate.ofEpochDay(asUnsignedInteger());
+    }
+
+    public LocalDateTime asDateTime() {
+        return asDateTime(0);
+    }
+
+    public LocalDateTime asDateTime(int scale) {
+        return ClickHouseValues
+                .convertToDateTime(asBigDecimal(ClickHouseChecker.between(scale, ClickHouseValues.PARAM_SCALE, 0, 9)));
+    }
+
     public float asFloat() {
         return getFloat(0);
     }

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataConfig.java

@@ -77,6 +77,11 @@ public ClickHouseRenameMethod getColumnRenameMethod() {
             return config.getColumnRenameMethod();
         }
 
+        @Override
+        public int getMaxMapperCache() {
+            return config.getMaxMapperCache();
+        }
+
         @Override
         public int getMaxQueuedBuffers() {
             return config.getMaxQueuedBuffers();
@@ -107,6 +112,11 @@ public boolean isUseBlockingQueue() {
             return config.isUseBlockingQueue();
         }
 
+        @Override
+        public boolean isUseCompilation() {
+            return config.isUseCompilation();
+        }
+
         @Override
         public boolean isUseObjectsInArray() {
             return config.isUseObjectsInArray();
@@ -128,6 +138,7 @@ public boolean isWidenUnsignedTypes() {
     static final int DEFAULT_READ_BUFFER_SIZE = DEFAULT_BUFFER_SIZE;
     static final int DEFAULT_WRITE_BUFFER_SIZE = DEFAULT_BUFFER_SIZE;
     static final int DEFAULT_MAX_BUFFER_SIZE = 128 * 1024;
+    static final int DEFAULT_MAX_MAPPER_CACHE = 100;
     static final int DEFAULT_MAX_QUEUED_BUFFERS = 512;
     static final int DEFAULT_BUFFER_QUEUE_VARIATION = 100;
 
@@ -138,6 +149,7 @@ public boolean isWidenUnsignedTypes() {
     static final boolean DEFAULT_REUSE_VALUE_WRAPPER = true;
     static final boolean DEFAULT_USE_BINARY_STRING = false;
     static final boolean DEFAULT_USE_BLOCKING_QUEUE = false;
+    static final boolean DEFAULT_USE_COMPILATION = false;
     static final boolean DEFAULT_USE_OBJECT_IN_ARRAY = false;
     static final boolean DEFAULT_WIDEN_UNSIGNED_TYPE = false;
 
@@ -255,6 +267,15 @@ default int getWriteBufferSize() {
         return DEFAULT_WRITE_BUFFER_SIZE;
     }
 
+    /**
+     * Gets maximum number of mappers can be cached.
+     *
+     * @return maximum number of mappers can be cached
+     */
+    default int getMaxMapperCache() {
+        return DEFAULT_MAX_MAPPER_CACHE;
+    }
+
     /**
      * Gets maximum number of buffers can be queued for processing.
      *
@@ -328,6 +349,15 @@ default boolean isUseBlockingQueue() {
         return DEFAULT_USE_BLOCKING_QUEUE;
     }
 
+    /**
+     * Checks whether compilation is used in object mapping and serialization.
+     *
+     * @return true if compilation is used; false otherwise
+     */
+    default boolean isUseCompilation() {
+        return DEFAULT_USE_COMPILATION;
+    }
+
     /**
      * Checks whether object(instead of primitive) is used in array.
      *