configurable layouts for open options (#2081)

Allows custom / external layout implementations to be passed into a file
reader through `VortexOpenOptions`.

Author: delta003

Cargo.lock

@@ -65,6 +65,12 @@ impl VortexOpenOptions {
         }
     }
 
+    /// Configure a layout context.
+    pub fn with_layouts(mut self, layout_ctx: LayoutContextRef) -> Self {
+        self.layout_ctx = layout_ctx;
+        self
+    }
+
     /// Configure a known file layout.
     ///
     /// If this is provided, then the Vortex file can be opened without performing any I/O.

vortex-file/src/open/mod.rs

@@ -17,6 +17,17 @@ impl LayoutContext {
         Self { layout_refs }
     }
 
+    pub fn with_layout(mut self, layout: LayoutEncodingRef) -> Self {
+        self.layout_refs.insert(layout.id(), layout);
+        self
+    }
+
+    pub fn with_layouts<E: IntoIterator<Item = LayoutEncodingRef>>(mut self, layouts: E) -> Self {
+        self.layout_refs
+            .extend(layouts.into_iter().map(|e| (e.id(), e)));
+        self
+    }
+
     pub fn lookup_layout(&self, id: LayoutId) -> Option<LayoutEncodingRef> {
         self.layout_refs.get(&id).cloned()
     }

vortex-layout/src/context.rs

@@ -45,6 +45,7 @@ vortex-sampling-compressor = { workspace = true }
 vortex-scalar = { workspace = true, default-features = true }
 vortex-scan = { workspace = true, default-features = true }
 vortex-zigzag = { workspace = true }
+vortex-layout = { workspace = true }
 
 [features]
 tokio = ["vortex-io/tokio", "vortex-file/tokio"]

vortex/Cargo.toml

@@ -2,8 +2,8 @@ pub use vortex_array::*;
 pub use {
     vortex_buffer as buffer, vortex_datetime_dtype as datetime_dtype, vortex_dtype as dtype,
     vortex_error as error, vortex_expr as expr, vortex_file as file,
-    vortex_flatbuffers as flatbuffers, vortex_io as io, vortex_ipc as ipc, vortex_mask as mask,
-    vortex_proto as proto, vortex_sampling_compressor as sampling_compressor,
+    vortex_flatbuffers as flatbuffers, vortex_io as io, vortex_ipc as ipc, vortex_layout as layout,
+    vortex_mask as mask, vortex_proto as proto, vortex_sampling_compressor as sampling_compressor,
     vortex_scalar as scalar, vortex_scan as scan,
 };