preservedims in tables

src/DimensionalData.jl

@@ -108,6 +108,7 @@ include("tables.jl")
 include("plotrecipes.jl")
 include("utils.jl")
 include("set.jl")
+include("opaque.jl")
 include("groupby.jl")
 include("precompile.jl")
 include("interface_tests.jl")

src/dimindices.jl

@@ -300,11 +300,13 @@ struct DimSlices{T,N,D<:Tuple{Vararg{Dimension}},P} <: AbstractDimArrayGenerator
     dims::D
 end
 DimSlices(x; dims, drop=true) = DimSlices(x, dims; drop)
-function DimSlices(x, dims; drop=true)
+DimSlices(x, dim; kw...) = DimSlices(x, (dim,); kw...)
+function DimSlices(x, dims::Tuple; drop=true)
+    dims1 = DD.dims(x, dims)
     newdims = if length(dims) == 0
         map(d  -> rebuild(d, :), DD.dims(x))
     else
-        dims
+        dims1
     end 
     inds = map(newdims) do d
         rebuild(d, first(d))
@@ -329,14 +331,14 @@ end
     I = (i1, i2, Is...)
     @boundscheck checkbounds(ds, I...)
     D = map(dims(ds), I) do d, i
-        rebuild(d, d[i])
+        rebuild(d, i)
     end
     return view(ds._data, D...)
 end
 # Dispatch to avoid linear indexing in multidimensional DimIndices
 @propagate_inbounds function Base.getindex(ds::DimSlices{<:Any,1}, i::Integer)
-    d = dims(ds, 1)
-    return view(ds._data, rebuild(d, d[i]))
+    d1 = dims(ds, 1)
+    return view(ds._data, rebuild(d1, i))
 end
 
 # Extends the dimensions of any `AbstractBasicDimArray`

src/groupby.jl

@@ -83,18 +83,6 @@
 end
 Base.alignment(io::IO, s::DimSummariser) = (textwidth(sprint(show, s)), 0)
 
-# An array that doesn't know what it holds, to simplify dispatch
-# It can also hold something that is not an AbstractArray itself.
-struct OpaqueArray{T,N,P} <: AbstractArray{T,N}
-    parent::P
-end
-OpaqueArray(A::P) where P<:AbstractArray{T,N} where {T,N} = OpaqueArray{T,N,P}(A)
-OpaqueArray(st::P) where P<:AbstractDimStack{<:Any,T,N} where {T,N} = OpaqueArray{T,N,P}(st)
-
-Base.size(A::OpaqueArray) = size(A.parent)
-Base.getindex(A::OpaqueArray, args...) = Base.getindex(A.parent, args...)
-Base.setindex!(A::OpaqueArray, args...) = Base.setindex!(A.parent, args...)
-
 
 abstract type AbstractBins <: Function end
 
@@ -247,7 +235,7 @@
 
 Group some data along the time dimension:
 
 ```jldoctest groupby; setup = :(using Random; Random.seed!(123))
 julia> using DimensionalData, Dates
 
 julia> A = rand(X(1:0.1:20), Y(1:20), Ti(DateTime(2000):Day(3):DateTime(2003)));
@@ -272,7 +260,7 @@
 
 And take the mean:
 
 ```jldoctest groupby; setup = :(using Statistics)
 julia> groupmeans = mean.(groups) # Take the monthly mean
 ┌ 12-element DimArray{Float64, 1} ┐
 ├─────────────────────────────────┴─────────────────────────────── dims ┐
@@ -295,13 +283,13 @@
 `.-` rather than `-`. This is because the size of the arrays to not
 match after application of `mean`.
 
 ```jldoctest groupby
 julia> map(.-, groupby(A, Ti=>month), mean.(groupby(A, Ti=>month), dims=Ti));
 ```
 
 Or do something else with Y:
 
 ```jldoctest groupby
 julia> groupmeans = mean.(groupby(A, Ti=>month, Y=>isodd))
 ┌ 12×2 DimArray{Float64, 2} ┐
 ├───────────────────────────┴────────────────────────────────────── dims ┐

src/opaque.jl

@@ -0,0 +1,19 @@
+# OpaqueArray is an array that doesn't know what it holds, to simplify dispatch.
+# One key property is that `parent(A::OpaqueArray)` returns the `OpaqueArray` `A`
+# not the array it holds. 
+# 
+# It is often used here to hide dimensional arrays that may be generated lazily,
+# To force them to act like simple Arrays without dimensional properties.
+#
+# OpaqueArray can also hold something that is not an AbstractArray itself.
+struct OpaqueArray{T,N,P} <: AbstractArray{T,N}
+    parent::P
+end
+OpaqueArray(A::P) where P<:AbstractArray{T,N} where {T,N} = OpaqueArray{T,N,P}(A)
+OpaqueArray(st::P) where P<:AbstractDimStack{<:Any,T,N} where {T,N} = OpaqueArray{T,N,P}(st)
+
+Base.size(A::OpaqueArray) = size(A.parent)
+Base.getindex(A::OpaqueArray, I::Union{StandardIndices,Not{<:StandardIndices}}...) = 
+    Base.getindex(A.parent, I...)
+Base.setindex!(A::OpaqueArray, I::Union{StandardIndices,Not{<:StandardIndices}}...) = 
+    Base.setindex!(A.parent, I...)

src/tables.jl

@@ -57,39 +57,54 @@
 
 # Keywords
 - `mergedims`: Combine two or more dimensions into a new dimension.
-- `layersfrom`: Treat a dimension of an `AbstractDimArray` as layers of an `AbstractDimStack`.
+- `preservedims`: Preserve one or more dimensions from flattening into the table. 
+    `DimArray`s of views with these dimensions will be present in the layer column,
+    rather than scalar values.
+- `layersfrom`: Treat a dimension of an `AbstractDimArray` as layers of an `AbstractDimStack`
+    by specifying a dimension to use as layers.
 
 # Example
 
-```jldoctest
+Here we generate a GeoInterface.jl compatible table with `:geometry` 
+column made of `(X, Y)` points, and data columns from `:band` slices.
+
+```jldoctest tables
 julia> using DimensionalData, Tables
 
-julia> a = DimArray(ones(16, 16, 3), (X, Y, Dim{:band}))
-┌ 16×16×3 DimArray{Float64, 3} ┐
-├──────────────────────── dims ┤
-  ↓ X, → Y, ↗ band
-└──────────────────────────────┘
-[:, :, 1]
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  …  1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  …  1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  …  1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0     1.0  1.0  1.0  1.0  1.0  1.0  1.0
- 1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  …  1.0  1.0  1.0  1.0  1.0  1.0  1.0
-
-julia> 
+julia> A = ones(X(4), Y(3), Dim{:band}('a':'d'); name=:data);
 
+julia> DimTable(A; layersfrom=:band, mergedims=(X, Y)=>:geometry)
+DimTable with 12 rows, 5 columns, and schema:
+ :geometry  Tuple{Int64, Int64}
+ :band_a    Float64
+ :band_b    Float64
+ :band_c    Float64
+ :band_d    Float64
 ```
+
+And here bands for each X/Y position are kept as vectors, using `preservedims`. 
+This may be useful if e.g. bands are color components of spectral images.
+
+```jldoctest tables
+julia> DimTable(A; preservedims=:band)
+DimTable with 12 rows, 3 columns, and schema:
+ :X     …  Int64
+ :Y        Int64
+ :data     DimVector{Float64, Tuple{Dim{:band, Categorical{Char, StepRange{Char, Int64}, ForwardOrdered, NoMetadata}}}, Tuple{X{NoLookup{UnitRange{Int64}}}, Y{NoLookup{UnitRange{Int64}}}}, SubArray{Float64, 1, Array{Float64, 3}, Tuple{Int64, 
+Int64, Slice{OneTo{Int64}}}, true}, Symbol, NoMetadata} (alias for DimArray{Float64, 1, Tuple{Dim{:band, Categorical{Char, StepRange{Char, Int64}, ForwardOrdered, NoMetadata}}}, Tuple{X{NoLookup{UnitRange{Int64}}}, Y{NoLookup{UnitRange{Int64}}}}, SubArray{Float64, 1, Array{Float64, 3}, Tuple{Int64, Int64, Base.Slice{Base.OneTo{Int64}}}, true}, Symbol, NoMetadata})
+````
+
+With no keywords, all data is flattened to a single column, 
+and all dimensions are included as columns, unrolled to match
+the length of the data.
+
+```jldoctest tables
+julia> DimTable(A)
+DimTable with 48 rows, 4 columns, and schema:
+ :X     Int64
+ :Y     Int64
+ :band  Char
+ :data  Float64
 """
 struct DimTable <: AbstractDimTable
     parent::Union{AbstractDimArray,AbstractDimStack}
@@ -98,8 +113,19 @@
     dimarraycolumns::Vector{AbstractVector}
 end
 
-function DimTable(s::AbstractDimStack; mergedims=nothing)
+function DimTable(s::AbstractDimStack; 
+    mergedims=nothing,
+    preservedims=nothing,
+)
     s = isnothing(mergedims) ? s : DD.mergedims(s, mergedims)
+    s = if isnothing(preservedims) 
+        s
+    else
+        maplayers(s) do A
+            S = DimSlices(A; dims=otherdims(A, preservedims))
+            dimconstructor(dims(S))(OpaqueArray(S), dims(S))
+        end
+    end
     dimcolumns = collect(_dimcolumns(s))
     dimarraycolumns = if hassamedims(s)
         map(vec, layers(s))
@@ -109,7 +135,9 @@
     keys = collect(_colnames(s))
     return DimTable(s, keys, dimcolumns, dimarraycolumns)
 end
-function DimTable(xs::Vararg{AbstractDimArray}; layernames=nothing, mergedims=nothing)
+function DimTable(xs::Vararg{AbstractDimArray}; 
+    layernames=nothing, mergedims=nothing, preservedims=nothing
+)
     # Check that dims are compatible
     comparedims(xs...)
 
@@ -118,6 +146,14 @@
 
     # Construct dimension and array columns with DimExtensionArray
     xs = isnothing(mergedims) ? xs : map(x -> DimensionalData.mergedims(x, mergedims), xs)
+    xs = if isnothing(preservedims)
+        xs
+    else
+        map(xs) do A
+            S = DimSlices(A; dims=otherdims(A, preservedims))
+            dimconstructor(dims(S))(OpaqueArray(S), dims(S))
+        end
+    end
     dims_ = dims(first(xs))
     dimcolumns = collect(_dimcolumns(dims_))
     dimnames = collect(map(name, dims_))
@@ -127,7 +163,9 @@
     # Return DimTable
     return DimTable(first(xs), colnames, dimcolumns, dimarraycolumns)
 end
-function DimTable(x::AbstractDimArray; layersfrom=nothing, mergedims=nothing)
+function DimTable(x::AbstractDimArray; 
+    layersfrom=nothing, mergedims=nothing, kw...
+)
     if !isnothing(layersfrom) && any(hasdim(x, layersfrom))
         d = dims(x, layersfrom)
         nlayers = size(x, d)
@@ -137,10 +175,10 @@
         else
             Symbol.(("$(name(d))_$i" for i in 1:nlayers))
         end
-        return DimTable(layers..., layernames=layernames, mergedims=mergedims)
+        return DimTable(layers...; layernames, mergedims, kw...)
     else
         s = name(x) == NoName() ? DimStack((;value=x)) : DimStack(x)
-        return  DimTable(s, mergedims=mergedims)
+        return  DimTable(s; mergedims, kw...)
     end
 end
 

test/stack.jl

@@ -3,7 +3,7 @@ using DimensionalData, Test, LinearAlgebra, Statistics, ConstructionBase, Random
 using DimensionalData: data
 using DimensionalData: Sampled, Categorical, AutoLookup, NoLookup, Transformed,
     Regular, Irregular, Points, Intervals, Start, Center, End,
-    Metadata, NoMetadata, ForwardOrdered, ReverseOrdered, Unordered, layers, basedims
+    Metadata, NoMetadata, ForwardOrdered, ReverseOrdered, Unordered, layers, basedims, metadata
 
 A = [1.0 2.0 3.0;
      4.0 5.0 6.0]

test/tables.jl

@@ -1,4 +1,4 @@
-using DimensionalData, IteratorInterfaceExtensions, TableTraits, Tables, Test, DataFrames
+using DimensionalData, IteratorInterfaceExtensions, TableTraits, Tables, Test, DataFrames, Dates
 
 using DimensionalData.Lookups, DimensionalData.Dimensions
 using DimensionalData: DimTable, DimExtensionArray
@@ -154,3 +154,19 @@ end
     @test Tables.columnnames(t3) == (:dimensions, :layer1, :layer2, :layer3)
     @test Tables.columnnames(t4) == (:band, :geometry, :value)
 end
+
+@testset "DimTable preservedims" begin
+    x, y, t = X(1.0:32.0), Y(1.0:10.0), Ti(DateTime.([2001, 2002, 2003]))
+    st = DimStack([rand(x, y, t; name) for name in [:a, :b, :c]])
+    A = rand(x, y, Dim{:band}(1:3); name=:vals)
+    t1 = DimTable(st, preservedims=(X, Y))
+    a3 = Tables.getcolumn(t1, :a)[3]
+    @test Tables.columnnames(t1) == propertynames(t1) == (:Ti, :a, :b, :c)
+    @test a3 == st.a[Ti=3]
+    @test dims(a3) == dims(st, (X, Y))
+    t2 = DimTable(A, preservedims=:band)
+    val10 = Tables.getcolumn(t2, :vals)[10]
+    @test Tables.columnnames(t2) == propertynames(t2) == (:X, :Y, :vals)
+    @test val10 == A[X(10), Y(1)]
+    @test dims(val10) == dims(A, (:band,))
+end