Implement Load Enumerations Request

Author: SarantopoulosKon

src/TileDBClient/TileDBClient.ts

@@ -7,13 +7,24 @@ import {
   OrganizationApi,
   NotebookApi,
   TasksApi,
-  UserApi
+  UserApi,
+  LoadEnumerationsRequest
 } from '../v1';
 import UDF from '../UDF';
 import Sql from '../Sql';
 import Groups from '../Groups';
-import { ConfigurationParameters, Configuration, Layout } from '../v2';
+import {
+  ConfigurationParameters,
+  Configuration,
+  Layout,
+  Datatype
+} from '../v2';
 import TileDBQuery from '../TileDBQuery';
+import bufferToData from '../utils/bufferToData';
+import groupValuesByOffsetBytes from '../utils/groupValuesByOffsetBytes';
+import convertToArray from '../utils/convertToArray';
+import getByteLengthOfDatatype from '../utils/getByteLengthOfDatatype';
+import concatChars from '../utils/concatChars';
 
 interface NotebookOrFileDimensions {
   contents: number[];
@@ -94,6 +105,88 @@ class TileDBClient {
     return this.ArrayApi.getArrayMetadata(namespace, array, options);
   }
 
+  public async loadEnumerationsRequest(
+    namespace: string,
+    array: string,
+    config: {
+      enumerations: string[];
+      enumerationsMaxSize?: string;
+      enumerationsTotalSize?: string;
+    }
+  ) {
+    const {
+      enumerations,
+      enumerationsMaxSize = '10485760',
+      enumerationsTotalSize = '52428800'
+    } = config;
+    const loadEnumerationsOptions: LoadEnumerationsRequest = {
+      config: {
+        entries: [
+          {
+            key: 'sm.enumerations_max_size',
+            value: enumerationsMaxSize
+          },
+          {
+            key: 'sm.enumerations_max_total_size',
+            value: enumerationsTotalSize
+          }
+        ]
+      },
+      enumerations: enumerations
+    };
+    const response = await this.ArrayApi.loadEnumerations(
+      namespace,
+      array,
+      loadEnumerationsOptions
+    );
+
+    const resultPromises = response.data.enumerations.map(async enumeration => {
+      const { type, data, name, offsets } = enumeration;
+      // Data is returned as array of numbers, convert it to buffer
+      const dataBuffer = Uint8Array.from(data).buffer;
+      let values: any = bufferToData(dataBuffer, type as Datatype);
+
+      // In case of var-length data, use offsets to get results
+      if (offsets && offsets.length) {
+        /**
+         * Convert offsets from uint8 array to uint64
+         * Uint8 array: 0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0
+         * Uint64 array: 0, 3, 8
+         */
+        const offsetsBuffer = Uint8Array.from(offsets).buffer;
+        const byteOffsets = Array.from(new BigUint64Array(offsetsBuffer));
+        // Get how many bytes this type is
+        const BYTE_PER_ELEMENT = getByteLengthOfDatatype(type as Datatype);
+        /**
+         * Convert offsets by bytes to offsets by element,
+         * since some primitives are more than 1 byte long.
+         */
+        const offsetsAsNumbers = byteOffsets.map(o =>
+          Number(o / BigInt(BYTE_PER_ELEMENT))
+        );
+
+        const groupedValues = await groupValuesByOffsetBytes(
+          convertToArray(values),
+          offsetsAsNumbers
+        );
+
+        const valueIsString = typeof values === 'string';
+        values = valueIsString
+          ? concatChars(groupedValues as string[][])
+          : (groupedValues as number[][] | bigint[][]);
+      }
+
+      return {
+        name: name,
+        type: type,
+        values
+      };
+    });
+    const result = await Promise.all(resultPromises);
+
+    return result;
+  }
+
   public arrayActivity(
     namespace: string,
     array: string,

src/TileDBQuery/TileDBQuery.ts

@@ -1,7 +1,8 @@
 import dataToQuery from '../utils/dataToQuery';
 import capnpQueryDeSerializer from '../utils/deserialization/capnpQueryDeSerializer';
-import { ArrayApi, ArraySchema } from '../v1';
+import { ArrayApi } from '../v1';
 import {
+  ArraySchema,
   AttributeBufferHeader,
   Configuration,
   ConfigurationParameters,

src/fixtures/attributes.ts

@@ -1,4 +1,4 @@
-import { Datatype } from '../v1';
+import { Datatype } from '../v2';
 
 export const fixedLenAttributesSchema = [
   {

src/fixtures/attributesAndDomains.ts

@@ -1,4 +1,4 @@
-import { Attribute, Dimension, Datatype } from '../v1';
+import { Attribute, Dimension, Datatype } from '../v2';
 
 const attributesAndDomains: Array<Dimension | Attribute> = [
   {

src/utils/attributeValuesToArrayBuffers/attributeValuesToArrayBuffers.ts

@@ -1,7 +1,6 @@
 import { AttributeValues } from '../../TileDBQuery';
-import { Attribute, Dimension } from '../../v1';
 import dataToArrayBuffer from '../dataToArrayBuffer';
-import { Datatype } from '../../v2';
+import { Datatype, Attribute, Dimension } from '../../v2';
 import mapToBigIntIfNeeded from '../mapToBigIntIfNeeded';
 
 interface ValueBuffer {

src/utils/dataToQueryWriter/dataToQueryWriter.ts

@@ -1,6 +1,5 @@
-import { Dimension } from '../../v1';
 import { ValueBuffers } from '../attributeValuesToArrayBuffers';
-import { Querystatus, Querytype } from '../../v2';
+import { Querystatus, Querytype, Dimension } from '../../v2';
 import getRanges from '../getRanges';
 import { QueryWrite } from '../../TileDBQuery';
 import flatten from '../flatten';

src/utils/getWriterBody/getWriterBody.ts

@@ -1,4 +1,5 @@
 import { ArraySchema } from '../../v1';
+import { Dimension, Attribute } from '../../v2';
 import dataToQueryWriter from '../dataToQueryWriter';
 import attributeValuesToArrayBuffers from '../attributeValuesToArrayBuffers';
 import capnpQuerySerializer from '../serialization/capnpQuerySerializer';
@@ -8,8 +9,8 @@ import { QueryWrite } from '../../TileDBQuery';
 const emptyArrayBuffer = new ArrayBuffer(0);
 
 const getWriterBody = (data: QueryWrite, arraySchema: ArraySchema) => {
-  const dimensions = arraySchema.domain.dimensions;
-  const attributes = arraySchema.attributes;
+  const dimensions = arraySchema.domain.dimensions as Dimension[];
+  const attributes = arraySchema.attributes as Attribute[];
   const valueBuffers = attributeValuesToArrayBuffers(
     data.values,
     dimensions,