Migrate away from deprecated TileDB C++ APIs (#1958)

* Migrate use of two deprecated APIs.

* Update tests to use the new delete_fragments API.

* Fix instance/static method conflict

* Migrate use of tiledb_array_create_with_key deprecated API.

* Migrate use of tiledb_array_consolidate_with_key deprecated API.

---------

Co-authored-by: Theodore Tsirpanis <[email protected]>

Author: kounelisagis

tiledb/libmetadata.pyx

@@ -400,11 +400,12 @@ cdef class Metadata:
                 ctx.__capsule__(), "ctx")
             tiledb_config_t* config_ptr = NULL
             tiledb_encryption_type_t key_type = TILEDB_NO_ENCRYPTION
-            void* key_ptr = NULL
+            const char* key_ptr = NULL
             uint32_t key_len = 0
             bytes bkey
             bytes buri = unicode_path(self.array.uri)
             str key = (<Array?>self.array).key
+            tiledb_error_t* err_ptr = NULL
 
         if config:
             config_ptr = <tiledb_config_t*>PyCapsule_GetPointer(
@@ -416,19 +417,28 @@ cdef class Metadata:
             else:
                 bkey = bytes(self.array.key)
             key_type = TILEDB_AES_256_GCM
-            key_ptr = <void *> PyBytes_AS_STRING(bkey)
+            key_ptr = <const char *> PyBytes_AS_STRING(bkey)
             #TODO: unsafe cast here ssize_t -> uint64_t
             key_len = <uint32_t> PyBytes_GET_SIZE(bkey)
 
+            rc = tiledb_config_alloc(&config_ptr, &err_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+
+            rc = tiledb_config_set(config_ptr, "sm.encryption_type", "AES_256_GCM", &err_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+
+            rc = tiledb_config_set(config_ptr, "sm.encryption_key", key_ptr, &err_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+
         cdef const char* buri_ptr = <const char*>buri
 
         with nogil:
-            rc = tiledb_array_consolidate_with_key(
+            rc = tiledb_array_consolidate(
                     ctx_ptr,
                     buri_ptr,
-                    key_type,
-                    key_ptr,
-                    key_len,
                     config_ptr)
         if rc != TILEDB_OK:
             _raise_ctx_err(ctx_ptr, rc)

tiledb/libtiledb.pxd

@@ -651,9 +651,8 @@ cdef extern from "tiledb/tiledb.h":
         uint32_t key_length,
         tiledb_config_t* config) nogil
 
-    int tiledb_array_delete_array(
+    int tiledb_array_delete(
         tiledb_ctx_t* ctx,
-        tiledb_array_t* array,
         const char* uri) nogil
 
     # Query
@@ -846,14 +845,6 @@ cdef extern from "tiledb/tiledb.h":
         const char* uri,
         const tiledb_array_schema_t* array_schema) nogil
 
-    int tiledb_array_create_with_key(
-        tiledb_ctx_t* ctx,
-        const char* uri,
-        const tiledb_array_schema_t* array_schema,
-        tiledb_encryption_type_t key_type,
-        const void* key,
-        unsigned int key_len) nogil
-
     int tiledb_array_is_open(
         tiledb_ctx_t* ctx,
         tiledb_array_t* array,
@@ -864,17 +855,15 @@ cdef extern from "tiledb/tiledb.h":
         const char* array_path,
         tiledb_config_t* config) nogil
 
-    int tiledb_array_consolidate_with_key(
+    int tiledb_array_delete_fragments(
         tiledb_ctx_t* ctx,
+        tiledb_array_t* array,
         const char* uri,
-        tiledb_encryption_type_t key_type,
-        const void* key_ptr,
-        unsigned int key_len,
-        tiledb_config_t* config) nogil
+        uint64_t timestamp_start,
+        uint64_t timestamp_end)
 
-    int tiledb_array_delete_fragments(
+    int tiledb_array_delete_fragments_v2(
         tiledb_ctx_t* ctx,
-        tiledb_array_t* array,
         const char* uri,
         uint64_t timestamp_start,
         uint64_t timestamp_end)

tiledb/libtiledb.pyx

@@ -940,9 +940,11 @@ cdef class Array(object):
 
         cdef bytes bkey
         cdef tiledb_encryption_type_t key_type = TILEDB_NO_ENCRYPTION
-        cdef void* key_ptr = NULL
+        cdef const char* key_ptr = NULL
         cdef unsigned int key_len = 0
 
+        cdef tiledb_config_t* config_ptr = NULL
+        cdef tiledb_error_t* err_ptr = NULL
         cdef int rc = TILEDB_OK
 
         if key is not None:
@@ -951,10 +953,25 @@ cdef class Array(object):
             else:
                 bkey = bytes(key)
             key_type = TILEDB_AES_256_GCM
-            key_ptr = <void *> PyBytes_AS_STRING(bkey)
+            key_ptr = <const char *> PyBytes_AS_STRING(bkey)
             #TODO: unsafe cast here ssize_t -> uint64_t
             key_len = <unsigned int> PyBytes_GET_SIZE(bkey)
 
+            rc = tiledb_config_alloc(&config_ptr, &err_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+
+            rc = tiledb_config_set(config_ptr, "sm.encryption_type", "AES_256_GCM", &err_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+
+            rc = tiledb_config_set(config_ptr, "sm.encryption_key", key_ptr, &err_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+            rc = tiledb_ctx_alloc(config_ptr, &ctx_ptr)
+            if rc != TILEDB_OK:
+                _raise_ctx_err(ctx_ptr, rc)
+
         if overwrite:
             if object_type(uri) == "array":
                 if uri.startswith("file://") or "://" not in uri:
@@ -971,7 +988,7 @@ cdef class Array(object):
                                 "object to argument ctx")
             ctx_ptr = safe_ctx_ptr(ctx)
         with nogil:
-            rc = tiledb_array_create_with_key(ctx_ptr, uri_ptr, schema_ptr, key_type, key_ptr, key_len)
+            rc = tiledb_array_create(ctx_ptr, uri_ptr, schema_ptr)
         if rc != TILEDB_OK:
             _raise_ctx_err(ctx_ptr, rc)
         return
@@ -1246,7 +1263,8 @@ cdef class Array(object):
             _raise_ctx_err(ctx_ptr, rc)
         return Enumeration.from_capsule(self.ctx, PyCapsule_New(enum_ptr, "enum", NULL))
 
-    def delete_fragments(self, timestamp_start, timestamp_end):
+    @staticmethod
+    def delete_fragments(uri, timestamp_start, timestamp_end, ctx=None):
         """
         Delete a range of fragments from timestamp_start to timestamp_end.
         The array needs to be opened in 'm' mode as shown in the example below.
@@ -1270,24 +1288,31 @@ cdef class Array(object):
         ...     A[:]
         array([1., 1., 1., 1.])
 
-        >>> with tiledb.open(path, 'm') as A:
-        ...     A.delete_fragments(2, 2)
+        >>> tiledb.Array.delete_fragments(path, 2, 2)
 
         >>> with tiledb.open(path, 'r') as A:
         ...     A[:]
         array([0., 0., 0., 0.])
 
         """
-        cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(self.ctx)
-        cdef tiledb_array_t* array_ptr = self.ptr
-        cdef tiledb_query_t* query_ptr = NULL
-        cdef bytes buri = self.uri.encode('UTF-8')
+        # If uri is an instance of Array (user calls the old instance method), issue a warning
+        if isinstance(uri, Array):
+            warnings.warn(
+                "The `tiledb.Array.delete_fragments` instance method is deprecated. Use the static method with the same name instead.",
+                DeprecationWarning,
+            )
+            uri = uri.uri
+
+        if not ctx:
+            ctx = default_ctx()
+
+        cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(ctx)
+        cdef bytes buri = uri.encode('UTF-8')
 
         cdef int rc = TILEDB_OK
 
-        rc = tiledb_array_delete_fragments(
+        rc = tiledb_array_delete_fragments_v2(
                 ctx_ptr,
-                array_ptr,
                 buri,
                 timestamp_start,
                 timestamp_end
@@ -1324,12 +1349,10 @@ cdef class Array(object):
 
         cdef tiledb_ctx_t* ctx_ptr = safe_ctx_ptr(ctx)
         cdef bytes buri = uri.encode('UTF-8')
-        cdef ArrayPtr preload_ptr = preload_array(uri, 'm', None, None)
-        cdef tiledb_array_t* array_ptr = preload_ptr.ptr
 
         cdef int rc = TILEDB_OK
 
-        rc = tiledb_array_delete_array(ctx_ptr, array_ptr, buri)
+        rc = tiledb_array_delete(ctx_ptr, buri)
         if rc != TILEDB_OK:
             _raise_ctx_err(ctx_ptr, rc)
 
@@ -3627,22 +3650,35 @@ def _consolidate_timestamp(uri, key=None, config=None, ctx=None, timestamp=None)
     cdef:
         bytes bkey
         tiledb_encryption_type_t key_type = TILEDB_NO_ENCRYPTION
-        void* key_ptr = NULL
+        const char* key_ptr = NULL
         unsigned int key_len = 0
+        tiledb_error_t* err_ptr = NULL
 
     if key is not None:
         if isinstance(key, str):
             bkey = key.encode('ascii')
         else:
             bkey = bytes(key)
         key_type = TILEDB_AES_256_GCM
-        key_ptr = <void *> PyBytes_AS_STRING(bkey)
+        key_ptr = <const char *> PyBytes_AS_STRING(bkey)
         #TODO: unsafe cast here ssize_t -> uint64_t
         key_len = <unsigned int> PyBytes_GET_SIZE(bkey)
+    
+        rc = tiledb_config_alloc(&config_ptr, &err_ptr)
+        if rc != TILEDB_OK:
+            _raise_ctx_err(ctx_ptr, rc)
+
+        rc = tiledb_config_set(config_ptr, "sm.encryption_type", "AES_256_GCM", &err_ptr)
+        if rc != TILEDB_OK:
+            _raise_ctx_err(ctx_ptr, rc)
+
+        rc = tiledb_config_set(config_ptr, "sm.encryption_key", key_ptr, &err_ptr)
+        if rc != TILEDB_OK:
+            _raise_ctx_err(ctx_ptr, rc)
 
     with nogil:
-        rc = tiledb_array_consolidate_with_key(
-            ctx_ptr, array_uri_ptr, key_type, key_ptr, key_len, config_ptr)
+        rc = tiledb_array_consolidate(
+            ctx_ptr, array_uri_ptr, config_ptr)
     if rc != TILEDB_OK:
         _raise_ctx_err(ctx_ptr, rc)
     return uri

tiledb/tests/test_fragments.py

@@ -704,13 +704,12 @@ def write_fragments(target_path, dshape, num_writes):
         if use_timestamps:
             assert frags.timestamp_range == ts
 
-        with tiledb.open(path, "m") as A:
-            if use_timestamps:
-                A.delete_fragments(3, 6)
-            else:
-                A.delete_fragments(
-                    frags.timestamp_range[2][0], frags.timestamp_range[5][1]
-                )
+        if use_timestamps:
+            tiledb.Array.delete_fragments(path, 3, 6)
+        else:
+            tiledb.Array.delete_fragments(
+                path, frags.timestamp_range[2][0], frags.timestamp_range[5][1]
+            )
 
         frags = tiledb.array_fragments(path)
         assert len(frags) == 6
@@ -755,13 +754,12 @@ def test_delete_fragments_with_schema_evolution(self, use_timestamps):
             assert_array_equal(A[:]["a1"], ts2_data)
             assert_array_equal(A[:]["a2"], ts2_data)
 
-        with tiledb.open(path, "m") as A:
-            if use_timestamps:
-                A.delete_fragments(2, 2)
-            else:
-                A.delete_fragments(
-                    frags.timestamp_range[1][0], frags.timestamp_range[1][1]
-                )
+        if use_timestamps:
+            tiledb.Array.delete_fragments(path, 2, 2)
+        else:
+            tiledb.Array.delete_fragments(
+                path, frags.timestamp_range[1][0], frags.timestamp_range[1][1]
+            )
 
         frags = tiledb.array_fragments(path)
         assert len(frags) == 1

tiledb/tests/test_libtiledb.py

@@ -370,12 +370,10 @@ def write_fragments(target_path, dshape, num_writes):
             assert frags.timestamp_range == ts
 
         if use_timestamps:
-            with tiledb.open(path, "m") as arr:
-                arr.delete_fragments(3, 6)
+            tiledb.Array.delete_fragments(path, 3, 6)
         else:
             timestamps = [t[0] for t in tiledb.array_fragments(path).timestamp_range]
-            with tiledb.open(path, "m") as arr:
-                arr.delete_fragments(timestamps[2], timestamps[5])
+            tiledb.Array.delete_fragments(path, timestamps[2], timestamps[5])
 
         frags = tiledb.array_fragments(path)
         assert len(frags) == 6