Initial CSV test working

Author: shmuelhanoch

.gitignore

@@ -76,3 +76,4 @@ server/src/main/resources/transport/definitions/manifest.txt
 
 # JEnv
 .java-version
+.aider*

.idea/runConfigurations/Debug_Elasticsearch__node_2_.xml

@@ -483,6 +483,8 @@ interface BlockFactory {
         SortedSetOrdinalsBuilder sortedSetOrdinalsBuilder(SortedSetDocValues ordinals, int count);
 
         AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count);
+
+        DateRangeBuilder dateRangeBuilder(int count);
     }
 
     /**
@@ -603,4 +605,10 @@ interface AggregateMetricDoubleBuilder extends Builder {
 
         IntBuilder count();
     }
+
+    interface DateRangeBuilder extends Builder {
+        LongBuilder from();
+
+        LongBuilder to();
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
@@ -349,6 +351,98 @@ public Query rangeQuery(
                 context
             );
         }
+
+        public static class DateRangeDocValuesLoader extends BlockDocValuesReader.DocValuesBlockLoader {
+            private final String fieldName;
+
+            public DateRangeDocValuesLoader(String fieldName) {
+                this.fieldName = fieldName;
+            }
+
+            @Override
+            public Builder builder(BlockFactory factory, int expectedCount) {
+                return factory.longsFromDocValues(expectedCount);
+            }
+
+            @Override
+            public AllReader reader(LeafReaderContext context) throws IOException {
+                var docValues = context.reader().getBinaryDocValues(fieldName);
+                return new DateRangeDocValuesReader(docValues);
+                /*
+                if (docValues != null) {
+                    NumericDocValues singleton = DocValues.unwrapSingleton(docValues);
+                    if (singleton != null) {
+                        return new BlockDocValuesReader.SingletonLongs(singleton);
+                    }
+                    return new BlockDocValuesReader.Longs(docValues);
+                }
+                NumericDocValues singleton = context.reader().getNumericDocValues(fieldName);
+                if (singleton != null) {
+                    return new BlockDocValuesReader.SingletonLongs(singleton);
+                }
+                return new ConstantNullsReader(); */
+            }
+        }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            if (hasDocValues()) {
+                return new DateRangeDocValuesLoader(name());
+            }
+            throw new IllegalStateException("Cannot load blocks without doc values");
+        }
+    }
+
+    public static class DateRangeDocValuesReader extends BlockDocValuesReader {
+        private final BytesRef spare = new BytesRef();
+
+        private final BinaryDocValues numericDocValues;
+
+        public DateRangeDocValuesReader(BinaryDocValues numericDocValues) {
+            this.numericDocValues = numericDocValues;
+        }
+
+        @Override
+        protected int docId() {
+            return 0;
+        }
+
+        @Override
+        public String toString() {
+            return "";
+        }
+
+        @Override
+        public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset, boolean nullsFiltered)
+            throws IOException {
+            try (BlockLoader.DateRangeBuilder builder = factory.dateRangeBuilder(docs.count() - offset)) {
+                for (int i = offset; i < docs.count(); i++) {
+                    int doc = docs.get(i);
+                    read(doc, builder);
+                }
+                return builder.build();
+            }
+        }
+
+        @Override
+        public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException {
+            read(docId, (BlockLoader.DateRangeBuilder) builder);
+        }
+
+        private void read(int doc, BlockLoader.DateRangeBuilder builder) throws IOException {
+            // WRONG?
+            if (false == numericDocValues.advanceExact(doc)) {
+                builder.appendNull();
+                return;
+            }
+
+            BytesRef ref = numericDocValues.binaryValue();
+            var ranges = BinaryRangeUtil.decodeLongRanges(ref);
+            for (var range : ranges) {
+                builder.from().appendLong((long) range.getFrom());
+                builder.to().appendLong((long) range.to);
+            }
+        }
     }
 
     private final RangeType type;
@@ -420,10 +514,14 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
 
     private Range parseRange(XContentParser parser) throws IOException {
         final XContentParser.Token start = parser.currentToken();
-        if (fieldType().rangeType == RangeType.IP && start == XContentParser.Token.VALUE_STRING) {
-            return parseIpRangeFromCidr(parser);
-        }
+        return (start != XContentParser.Token.VALUE_STRING) ? parseGeneralRange(parser, start) : switch (fieldType().rangeType) {
+            case RangeType.IP -> parseIpRangeFromCidr(parser);
+            case RangeType.DATE -> parseDateRange(parser.text());
+            default -> parseGeneralRange(parser, start);
+        };
+    }
 
+    private Range parseGeneralRange(XContentParser parser, XContentParser.Token start) throws IOException {
         if (start != XContentParser.Token.START_OBJECT) {
             throw new DocumentParsingException(
                 parser.getTokenLocation(),
@@ -483,6 +581,14 @@ private static Range parseIpRangeFromCidr(final XContentParser parser) throws IO
         return new Range(RangeType.IP, range.lowerBound(), range.upperBound(), true, true);
     }
 
+    private static Range parseDateRange(String s) throws IOException {
+        var nums = s.split("-");
+        assert nums.length == 2 : "Expected two numbers in the date range string: " + s;
+        var from = Long.parseLong(nums[0].trim());
+        var to = Long.parseLong(nums[1].trim());
+        return new Range(RangeType.DATE, from, to, true, false);
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {

server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java

@@ -372,9 +372,15 @@ public SortedSetOrdinalBuilder appendOrd(int value) {
                 return new SortedSetOrdinalBuilder();
             }
 
+            @Override
             public BlockLoader.AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int expectedSize) {
                 return new AggregateMetricDoubleBlockBuilder(expectedSize);
             }
+
+            @Override
+            public BlockLoader.DateRangeBuilder dateRangeBuilder(int expectedSize) {
+                return new DateRangeBuilder(expectedSize);
+            }
         };
     }
 
@@ -582,4 +588,61 @@ public void close() {
 
         }
     }
+
+    public static class DateRangeBuilder implements BlockLoader.DateRangeBuilder {
+        private final LongBuilder from;
+        private final LongBuilder to;
+
+        DateRangeBuilder(int expectedSize) {
+            from = new LongBuilder(expectedSize);
+            to = new LongBuilder(expectedSize);
+        }
+
+        @Override
+        public BlockLoader.LongBuilder from() {
+            return null;
+        }
+
+        @Override
+        public BlockLoader.LongBuilder to() {
+            return null;
+        }
+
+        @Override
+        public BlockLoader.Block build() {
+            return null;
+        }
+
+        @Override
+        public BlockLoader.Builder appendNull() {
+            return null;
+        }
+
+        @Override
+        public BlockLoader.Builder beginPositionEntry() {
+            return null;
+        }
+
+        @Override
+        public BlockLoader.Builder endPositionEntry() {
+            return null;
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        private static class LongBuilder extends TestBlock.Builder implements BlockLoader.LongBuilder {
+            private LongBuilder(int expectedSize) {
+                super(expectedSize);
+            }
+
+            @Override
+            public BlockLoader.LongBuilder appendLong(long value) {
+                add(value);
+                return null;
+            }
+        }
+    };
 }

test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java

@@ -15,4 +15,5 @@ public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin {
 
     public static final FeatureFlag AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG = new FeatureFlag("esql_aggregate_metric_double");
     public static final FeatureFlag DENSE_VECTOR_FEATURE_FLAG = new FeatureFlag("esql_dense_vector");
+    public static final FeatureFlag DATE_RANGE_FEATURE_FLAG = new FeatureFlag("esql_date_range");
 }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java

@@ -310,7 +310,13 @@ public enum DataType {
     /**
      * Fields with this type are dense vectors, represented as an array of double values.
      */
-    DENSE_VECTOR(builder().esType("dense_vector").unknownSize());
+    DENSE_VECTOR(builder().esType("dense_vector").unknownSize()),
+
+    /**
+     * Fields with this type are used to represent a range of dates.
+     * This is an under-construction type, and is not yet fully supported.
+     */
+    DATE_RANGE(builder().esType("date_range").typeName("DATE_RANGE").estimatedSize(2 * Long.BYTES));
 
     /**
      * Types that are actively being built. These types are
@@ -331,7 +337,8 @@ public enum DataType {
      */
     public static final Map<DataType, FeatureFlag> UNDER_CONSTRUCTION = Map.ofEntries(
         Map.entry(AGGREGATE_METRIC_DOUBLE, EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG),
-        Map.entry(DENSE_VECTOR, EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG)
+        Map.entry(DENSE_VECTOR, EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG),
+        Map.entry(DATE_RANGE, EsqlCorePlugin.DATE_RANGE_FEATURE_FLAG)
     );
 
     private final String typeName;

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java

@@ -21,6 +21,7 @@
     requires org.elasticsearch.geo;
     requires org.elasticsearch.xcore;
     requires hppc;
+    requires com.ibm.icu;
 
     exports org.elasticsearch.compute;
     exports org.elasticsearch.compute.aggregation;

x-pack/plugin/esql/compute/src/main/java/module-info.java

@@ -481,6 +481,10 @@ public final AggregateMetricDoubleBlock newAggregateMetricDoubleBlock(
         return new AggregateMetricDoubleArrayBlock(min, max, sum, count);
     }
 
+    public DateRangeBlockBuilder newDateRangeBlockBuilder(int estimatedSize) {
+        return new DateRangeBlockBuilder(estimatedSize, this);
+    }
+
     /**
      * Returns the maximum number of bytes that a Block should be backed by a primitive array before switching to using BigArrays.
      */

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java

@@ -299,6 +299,12 @@ yield new AggregateMetricDoubleLiteral(
                     aggBlock.countBlock().getInt(offset)
                 );
             }
+            case DATE_RANGE -> {
+                DateRangeBlock b = (DateRangeBlock) block;
+                LongBlock fromBlock = b.getFromBlock();
+                LongBlock toBlock = b.getToBlock();
+                yield new DateRangeBlockBuilder.DateRangeLiteral(fromBlock.getLong(offset), toBlock.getLong(offset));
+            }
             case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]");
         };
     }