Add VFS support for tiledb:// URIs. (#5631)

This PR adds support to VFS for `tiledb://` URIs. It reads the
`rest.***` config options, and based on them, it sets up an S3 VFS that
points to Carrara's S3-compatible `v4/files` endpoints.

---
TYPE: FEATURE
DESC: Add VFS support for `tiledb://` URIs.

Author: teo-tsirpanis

test/src/unit-vfs.cc

@@ -389,19 +389,16 @@ TEST_CASE("VFS: copy_dir", "[vfs][copy_dir]") {
   }
 }
 
-using AllBackends = std::tuple<LocalFsTest, GCSTest, GSTest, S3Test, AzureTest>;
+using AllBackends =
+    std::tuple<LocalFsTest, GCSTest, GSTest, S3Test, AzureTest, TileDBFSTest>;
 TEMPLATE_LIST_TEST_CASE(
-    "VFS: URI semantics and file management", "[vfs][uri]", AllBackends) {
+    "VFS: URI semantics and file management", "[vfs][rest][uri]", AllBackends) {
   TestType fs({});
   if (!fs.is_supported()) {
     return;
   }
 
-  ThreadPool compute_tp(4);
-  ThreadPool io_tp(4);
-  Config config = set_config_params();
-  VFS vfs{
-      &g_helper_stats, g_helper_logger().get(), &compute_tp, &io_tp, config};
+  auto& vfs = fs.vfs_;
 
   URI path = fs.temp_dir_.add_trailing_slash();
 
@@ -792,11 +789,11 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") {
 }
 
 // Currently only local, S3, Azure and GCS are supported for VFS::ls_recursive.
-using TestBackends = std::tuple<LocalFsTest, S3Test, AzureTest, GCSTest>;
+using LsRecursiveBackends = std::tuple<LocalFsTest, S3Test, AzureTest, GCSTest>;
 TEMPLATE_LIST_TEST_CASE(
     "VFS: ls_filtered recursion enabled",
-    "[vfs][ls_filtered][ls_filtered_v2][recursion]",
-    TestBackends) {
+    "[vfs][ls_filtered][ls_filtered_v2][rest][recursion]",
+    LsRecursiveBackends) {
   TestType fs({10, 50});
   if (!fs.is_supported()) {
     return;
@@ -822,7 +819,7 @@ TEMPLATE_LIST_TEST_CASE(
 TEMPLATE_LIST_TEST_CASE(
     "VFS: ls_filtered non-recursive",
     "[vfs][ls_filtered][ls_filtered_v2]",
-    TestBackends) {
+    LsRecursiveBackends) {
   TestType fs({10});
   if (!fs.is_supported()) {
     return;
@@ -860,9 +857,11 @@ TEMPLATE_LIST_TEST_CASE(
   }
 }
 
-TEST_CASE("VFS: Throwing filters for ls_recursive", "[vfs][ls_recursive]") {
-  std::string prefix = GENERATE("s3://", "azure://", "gcs://", "gs://");
-  VFSTest vfs_test({0}, prefix);
+TEMPLATE_LIST_TEST_CASE(
+    "VFS: Throwing filters for ls_recursive",
+    "[vfs][ls_recursive]",
+    LsRecursiveBackends) {
+  TestType vfs_test({0});
   if (!vfs_test.is_supported()) {
     return;
   }
@@ -1005,7 +1004,7 @@ TEST_CASE(
     "Validate GCS service account impersonation",
     "[gcs][credentials][impersonation]") {
   ThreadPool thread_pool(2);
-  Config cfg = set_config_params(true);
+  Config cfg;
   std::string impersonate_service_account, target_service_account;
   std::vector<std::string> delegates;
 
@@ -1049,7 +1048,7 @@ TEST_CASE(
     "Validate GCS service account credentials",
     "[gcs][credentials][service-account]") {
   ThreadPool thread_pool(2);
-  Config cfg = set_config_params(true);
+  Config cfg;
   // The content of the credentials does not matter; it does not get parsed
   // until it is used in an API request, which we are not doing.
   std::string service_account_key = "{\"foo\": \"bar\"}";
@@ -1072,7 +1071,7 @@ TEST_CASE(
     "Validate GCS service account credentials with impersonation",
     "[gcs][credentials][service-account-and-impersonation]") {
   ThreadPool thread_pool(2);
-  Config cfg = set_config_params(true);
+  Config cfg;
   // The content of the credentials does not matter; it does not get parsed
   // until it is used in an API request, which we are not doing.
   std::string service_account_key = "{\"foo\": \"bar\"}";
@@ -1107,7 +1106,7 @@ TEST_CASE(
     "Validate GCS external account credentials",
     "[gcs][credentials][external-account]") {
   ThreadPool thread_pool(2);
-  Config cfg = set_config_params(true);
+  Config cfg;
   // The content of the credentials does not matter; it does not get parsed
   // until it is used in an API request, which we are not doing.
   std::string workload_identity_configuration = "{\"foo\": \"bar\"}";

test/src/unit.cc

@@ -1,4 +1,5 @@
 #define CATCH_CONFIG_RUNNER
+#include <test/support/src/helpers.h>
 #include <test/support/tdb_catch.h>
 
 #include <cstdlib>
@@ -11,15 +12,6 @@
 #include <crtdbg.h>
 #endif
 
-namespace tiledb {
-namespace test {
-
-// Command line arguments.
-int store_g_vfs(std::string&& vfs, std::vector<std::string> vfs_fs);
-
-}  // namespace test
-}  // namespace tiledb
-
 int main(const int argc, char** const argv) {
 #if defined(_MSC_VER)
   // We disable the following events on abort in CI environments:

test/support/src/helpers.cc

@@ -210,6 +210,10 @@ void throw_if_error(tiledb_ctx_t* ctx, capi_return_t thing) {
   }
 }
 
+bool is_g_vfs_enabled(const std::string& vfs) {
+  return vfs == "memfs" || vfs == g_vfs;
+}
+
 int store_g_vfs(std::string&& vfs, std::vector<std::string> vfs_fs) {
   if (!vfs.empty()) {
     if (std::find(vfs_fs.begin(), vfs_fs.end(), vfs) == vfs_fs.end()) {

test/support/src/helpers.h

@@ -118,6 +118,12 @@ typedef std::pair<tiledb_filter_type_t, int> Compressor;
 template <class T>
 using SubarrayRanges = std::vector<std::vector<T>>;
 
+/** Checks whether the given VFS is enabled by tiledb_unit's command line. */
+bool is_g_vfs_enabled(const std::string& vfs);
+
+// Command line arguments.
+int store_g_vfs(std::string&& vfs, std::vector<std::string> vfs_fs);
+
 /**
  * Throws if the return code is not OK.
  * For use in test setup for object allocation.