Update Core to 2.18 (#325)

* Regenerate bindings for 2.18.

* Bump Core to 2.18.0.

* Run nightly builds on release-2.18.

* Bump version to 5.8.0.

* Add APIs to delete fragments from arrays.

* Add safe handle types for the newly introduced handles.

`tiledb_channel_operator` does not have a free function so it does not need a safe handle.

* Add APIs to evolve enumerations.

* Add a `QueryField` type and simplify the query data buffer type validation.

* Add query aggregate APIs.

* Add documentation.

* Add an aggregate query example.

* Test that all expected file system backends are supported.

* Add a test for aggregate queries.

* Update documentation.

* Add one more assertion.

* Test min/max aggregates.

* Update documentation.

* Fix error handling.

* Test null count aggregate.

Author: teo-tsirpanis

.github/workflows/nightly.yml

@@ -29,7 +29,7 @@ jobs:
             platform: windows-x86_64
             BOOTSTRAP: ../bootstrap.ps1 -EnableS3 -EnableSerialization -EnableBuildDeps -EnableVcpkg
           - tag: dev
-        tag: [release-2.17, dev]
+        tag: [release-2.18, dev]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout TileDB
@@ -67,7 +67,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        tag: [release-2.17, dev]
+        tag: [release-2.18, dev]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout TileDB-CSharp
@@ -107,7 +107,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        tag: [release-2.17, dev]
+        tag: [release-2.18, dev]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout TileDB-CSharp

Directory.Packages.props

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
     <TileDBNativePackageName>TileDB.Native</TileDBNativePackageName>
-    <TileDBNativePackageVersion>[2.17.0,2.18.0)</TileDBNativePackageVersion>
+    <TileDBNativePackageVersion>[2.18.0,2.19.0)</TileDBNativePackageVersion>
 
     <!-- The DevelopmentBuild property switches to the locally built native packages.
     They have a different name to avoid publishing them by mistake, and to

examples/TileDB.CSharp.Example/ExampleAggregateQuery.cs

@@ -0,0 +1,84 @@
+using System;
+using System.IO;
+
+namespace TileDB.CSharp.Examples
+{
+    static class ExampleAggregateQuery
+    {
+        private static readonly string ArrayPath = ExampleUtil.MakeExamplePath("aggregate-array");
+        private static readonly Context Ctx = Context.GetDefault();
+
+        private static void CreateArray()
+        {
+            // Create array
+            var dim1 = Dimension.Create(Ctx, "rows", boundLower: 1, boundUpper: 4, extent: 4);
+            var dim2 = Dimension.Create(Ctx, "cols", boundLower: 1, boundUpper: 4, extent: 4);
+            var domain = new Domain(Ctx);
+            domain.AddDimension(dim1);
+            domain.AddDimension(dim2);
+            var array_schema = new ArraySchema(Ctx, ArrayType.Sparse);
+            var attr = new Attribute(Ctx, "a", DataType.Int32);
+            array_schema.AddAttribute(attr);
+            array_schema.SetDomain(domain);
+            array_schema.Check();
+
+            Array.Create(Ctx, ArrayPath, array_schema);
+        }
+
+        private static void WriteArray()
+        {
+            using (var array_write = new Array(Ctx, ArrayPath))
+            {
+                array_write.Open(QueryType.Write);
+                using (var query_write = new Query(array_write))
+                {
+                    query_write.SetLayout(LayoutType.GlobalOrder);
+                    query_write.SetDataBuffer("rows", new int[] { 1, 2 });
+                    query_write.SetDataBuffer("cols", new int[] { 1, 4 });
+                    query_write.SetDataBuffer("a", new int[] { 1, 2 });
+                    query_write.Submit();
+                    query_write.SetDataBuffer("rows", new int[] { 3 });
+                    query_write.SetDataBuffer("cols", new int[] { 3 });
+                    query_write.SetDataBuffer("a", new int[] { 3 });
+                    query_write.SubmitAndFinalize();
+                }
+                array_write.Close();
+            }
+        }
+
+        private static void ReadArray()
+        {
+            ulong[] count = { 0 };
+            long[] sum = { 0 };
+
+            using (var array_read = new Array(Ctx, ArrayPath))
+            {
+                array_read.Open(QueryType.Read);
+                using var query_read = new Query(array_read);
+                query_read.SetLayout(LayoutType.Unordered);
+                using var channel = query_read.GetDefaultChannel();
+                channel.ApplyAggregate(AggregateOperation.Count, "Count");
+                channel.ApplyAggregate(AggregateOperation.Unary(AggregateOperator.Sum, "a"), "Sum");
+                query_read.SetDataBuffer("Count", count);
+                query_read.SetDataBuffer("Sum", sum);
+                query_read.Submit();
+                array_read.Close();
+            }
+
+            Console.WriteLine($"Count: {count[0]}");
+            Console.WriteLine($"Sum: {sum[0]}");
+        }
+
+        public static void Run()
+        {
+            if (Directory.Exists(ArrayPath))
+            {
+                Directory.Delete(ArrayPath, true);
+            }
+
+            CreateArray();
+            WriteArray();
+            ReadArray();
+        }
+    }
+}

examples/TileDB.CSharp.Example/Program.cs

@@ -16,6 +16,7 @@ static void Main(string[] args)
             ExampleWritingDenseGlobal.Run();
             ExampleWritingSparseGlobal.Run();
             ExampleDataframe.Run();
+            ExampleAggregateQuery.Run();
 
             ExampleFile.RunLocal();
             // ExampleFile.RunCloud("tiledb_api_token", "tiledb_namespace", "new_cloud_array_name", "s3://bucket/prefix/");

scripts/generate-bindings/GenerateBindings.proj

@@ -18,22 +18,26 @@
     <InputFile Include="tiledb/tiledb_experimental.h" />
     <TraversePaths Include="$(InputDir)/**/*.h" />
     <RemapHandleType Include="attribute" />
-    <RemapHandleType Include="buffer" />
     <RemapHandleType Include="buffer_list" />
-    <RemapHandleType Include="config" />
+    <RemapHandleType Include="buffer" />
+    <RemapHandleType Include="channel_operation" />
+    <RemapHandleType Include="channel_operator" />
     <RemapHandleType Include="config_iter" />
+    <RemapHandleType Include="config" />
     <RemapHandleType Include="ctx" />
-    <RemapHandleType Include="dimension" />
     <RemapHandleType Include="dimension_label" />
+    <RemapHandleType Include="dimension" />
     <RemapHandleType Include="domain" />
     <RemapHandleType Include="enumeration" />
     <RemapHandleType Include="error" />
-    <RemapHandleType Include="filter" />
     <RemapHandleType Include="filter_list" />
+    <RemapHandleType Include="filter" />
     <RemapHandleType Include="group" />
+    <RemapHandleType Include="query_channel" />
+    <RemapHandleType Include="query_field" />
     <RemapHandleType Include="string" />
-    <RemapHandleType Include="vfs" />
     <RemapHandleType Include="vfs_fh" />
+    <RemapHandleType Include="vfs" />
     <Remap Include="@(RemapHandleType->'tiledb_%(Identity)_handle_t=tiledb_%(Identity)_t')" />
     <Remap Include="tiledb_experimental_query_status_details_t=tiledb_query_status_details_t" />
     <ExcludeDump Include="array_schema" />

sources/TileDB.CSharp/AggregateOperation.cs

@@ -0,0 +1,98 @@
+using TileDB.CSharp.Marshalling.SafeHandles;
+using TileDB.Interop;
+
+namespace TileDB.CSharp;
+
+/// <summary>
+/// Represents an aggregate operation.
+/// </summary>
+/// <remarks>
+/// This class abstracts the <c>tiledb_channel_operation_t</c> type of the TileDB C API.
+/// </remarks>
+public abstract class AggregateOperation
+{
+    // Prevent inheriting the class outside of this assembly.
+    private protected AggregateOperation() { }
+
+    /// <summary>
+    /// Creates a unary aggregate operation.
+    /// </summary>
+    /// <param name="op">The <see cref="AggregateOperator"/> to apply.</param>
+    /// <param name="fieldName">The name of the field to apply the operator to.</param>
+    public static AggregateOperation Unary(AggregateOperator op, string fieldName) => new UnaryAggregateOperation(op, fieldName);
+
+    /// <summary>
+    /// An <see cref="AggregateOperation"/> returning the number of values in the channel.
+    /// </summary>
+    public static AggregateOperation Count { get; } = new CountAggregateOperation();
+
+    internal abstract unsafe ChannelOperationHandle CreateHandle(Context ctx, Query q);
+
+    private sealed class UnaryAggregateOperation : AggregateOperation
+    {
+        private readonly AggregateOperator _operator;
+
+        private readonly string _fieldName;
+
+        public UnaryAggregateOperation(AggregateOperator op, string fieldName)
+        {
+            _operator = op;
+            _fieldName = fieldName;
+        }
+
+        internal override unsafe ChannelOperationHandle CreateHandle(Context ctx, Query q)
+        {
+            var handle = new ChannelOperationHandle();
+            bool successful = false;
+            tiledb_channel_operation_t* op = null;
+            try
+            {
+                using var ctxHandle = ctx.Handle.Acquire();
+                using var queryHandle = q.Handle.Acquire();
+                using var ms_fieldName = new MarshaledString(_fieldName);
+                ctx.handle_error(Methods.tiledb_create_unary_aggregate(ctxHandle, queryHandle, _operator.GetHandle(ctxHandle), ms_fieldName, &op));
+                successful = true;
+            }
+            finally
+            {
+                if (successful)
+                {
+                    handle.InitHandle(ctx, op);
+                }
+                else
+                {
+                    handle.SetHandleAsInvalid();
+                }
+            }
+            return handle;
+        }
+    }
+
+    private sealed class CountAggregateOperation : AggregateOperation
+    {
+        internal override unsafe ChannelOperationHandle CreateHandle(Context ctx, Query q)
+        {
+            var handle = new ChannelOperationHandle();
+            bool successful = false;
+            tiledb_channel_operation_t* op = null;
+            try
+            {
+                using var ctxHandle = ctx.Handle.Acquire();
+                ctx.handle_error(Methods.tiledb_aggregate_count_get(ctxHandle, &op));
+                successful = true;
+            }
+            finally
+            {
+                if (successful)
+                {
+                    handle.InitHandle(ctx, op);
+                }
+                else
+                {
+                    handle.SetHandleAsInvalid();
+                }
+            }
+            return handle;
+        }
+    }
+}

sources/TileDB.CSharp/AggregateOperator.cs

@@ -0,0 +1,78 @@
+using System;
+using TileDB.Interop;
+
+namespace TileDB.CSharp;
+
+/// <summary>
+/// Represents an aggregate operator that can be applied over a query field.
+/// </summary>
+/// <remarks>
+/// This class abstracts the <c>tiledb_channel_operator_t</c> type of the TileDB C API.
+/// </remarks>
+public abstract class AggregateOperator
+{
+    // Prevent inheriting the class outside of this assembly.
+    private protected AggregateOperator() { }
+
+    internal abstract unsafe tiledb_channel_operator_t* GetHandle(tiledb_ctx_t* ctx);
+
+    /// <summary>
+    /// Computes the sum of all values in the field.
+    /// </summary>
+    public static AggregateOperator Sum { get; } = new SimpleAggregateOperator(OperatorKind.Sum);
+
+    /// <summary>
+    /// Computes the minimum value in the field.
+    /// </summary>
+    public static AggregateOperator Min { get; } = new SimpleAggregateOperator(OperatorKind.Min);
+
+    /// <summary>
+    /// Computes the maximum value in the field.
+    /// </summary>
+    public static AggregateOperator Max { get; } = new SimpleAggregateOperator(OperatorKind.Max);
+
+    /// <summary>
+    /// Computes the mean of all values in the field.
+    /// </summary>
+    public static AggregateOperator Mean { get; } = new SimpleAggregateOperator(OperatorKind.Mean);
+
+    /// <summary>
+    /// Counts the number of null values in the field.
+    /// </summary>
+    public static AggregateOperator NullCount { get; } = new SimpleAggregateOperator(OperatorKind.NullCount);
+
+    private enum OperatorKind
+    {
+        Sum,
+        Min,
+        Max,
+        Mean,
+        NullCount
+    }
+
+    private sealed class SimpleAggregateOperator : AggregateOperator
+    {
+        private readonly OperatorKind _kind;
+
+        public SimpleAggregateOperator(OperatorKind kind)
+        {
+            _kind = kind;
+        }
+
+        internal override unsafe tiledb_channel_operator_t* GetHandle(tiledb_ctx_t* ctx)
+        {
+            tiledb_channel_operator_t* op;
+            var ret = _kind switch
+            {
+                OperatorKind.Sum => Methods.tiledb_channel_operator_sum_get(ctx, &op),
+                OperatorKind.Min => Methods.tiledb_channel_operator_min_get(ctx, &op),
+                OperatorKind.Max => Methods.tiledb_channel_operator_max_get(ctx, &op),
+                OperatorKind.Mean => Methods.tiledb_channel_operator_mean_get(ctx, &op),
+                OperatorKind.NullCount => Methods.tiledb_channel_operator_null_count_get(ctx, &op),
+                _ => throw new InvalidOperationException(),
+            };
+            ErrorHandling.ThrowOnError(ret);
+            return op;
+        }
+    }
+}

sources/TileDB.CSharp/Array.cs

@@ -737,6 +737,34 @@ public void DeleteMetadata(string key)
             _metadata.DeleteMetadata(key);
         }
 
+        /// <summary>
+        /// Deletes fragments from an array that fall within the given timestamp range.
+        /// </summary>
+        /// <param name="ctx">The <see cref="Context"/> to use.</param>
+        /// <param name="uri">The URI to the array.</param>
+        /// <param name="timestampStart">The start of the timestamp range, inclusive.</param>
+        /// <param name="timestampEnd">The end of the timestamp range, inclusive.</param>
+        public static void DeleteFragments(Context ctx, string uri, ulong timestampStart, ulong timestampEnd)
+        {
+            using var ms_uri = new MarshaledString(uri);
+            using var ctxHandle = ctx.Handle.Acquire();
+            ctx.handle_error(Methods.tiledb_array_delete_fragments_v2(ctxHandle, ms_uri, timestampStart, timestampEnd));
+        }
+
+        /// <summary>
+        /// Deletes fragments from an array that fall within the given timestamp range.
+        /// </summary>
+        /// <param name="ctx">The <see cref="Context"/> to use.</param>
+        /// <param name="uri">The URI to the array.</param>
+        /// <param name="fragmentUris">A list with the URIs of the fragments to delete.</param>
+        public static void DeleteFragments(Context ctx, string uri, IReadOnlyList<string> fragmentUris)
+        {
+            using var ms_uri = new MarshaledString(uri);
+            using var msc_fragmentUris = new MarshaledStringCollection(fragmentUris);
+            using var ctxHandle = ctx.Handle.Acquire();
+            ctx.handle_error(Methods.tiledb_array_delete_fragments_list(ctxHandle, ms_uri, (sbyte**)msc_fragmentUris.Strings, (nuint)msc_fragmentUris.Count));
+        }
+
         /// <summary>
         /// Gets metadata from the <see cref="Array"/>.
         /// </summary>

sources/TileDB.CSharp/ArraySchemaEvolution.cs

@@ -76,6 +76,19 @@ public void AddEnumeration(Enumeration enumeration)
             _ctx.handle_error(Methods.tiledb_array_schema_evolution_add_enumeration(ctxHandle, handle, enumHandle));
         }
 
+        /// <summary>
+        /// Adds an <see cref="Enumeration"/> to the <see cref="ArraySchemaEvolution"/>.
+        /// </summary>
+        /// <param name="enumeration">A fully constructed <see cref="Enumeration"/> that will be added to the schema.</param>
+        /// <seealso cref="DropEnumeration"/>
+        public void ExtendEnumeration(Enumeration enumeration)
+        {
+            using var ctxHandle = _ctx.Handle.Acquire();
+            using var handle = _handle.Acquire();
+            using var enumHandle = enumeration.Handle.Acquire();
+            _ctx.handle_error(Methods.tiledb_array_schema_evolution_extend_enumeration(ctxHandle, handle, enumHandle));
+        }
+
         /// <summary>
         /// Drops an enumeration with the given name from the <see cref="ArraySchemaEvolution"/>.
         /// </summary>