Remove SparseVarArray (#31)

* Removing SparseVarArray

* Remove kwargs for inservar!

* Delete most commented-out removals

* update JuliaFormatter for CI

* In progress: updating tests

* formatting fixes

* x86 fix

* Update tests
Remove tables awaiting #30

* Test starting from 1.7 for destructuring

* Test coverage

* formatting fixes

* Delete unused for coverage

* SparseArray coverage

* Formatting fixes

* x86 fixes (Int)

* Formatting fix

* Update readme and prep for v0.7

* Remove README entry on rowtables for now

* Prep release

Co-authored-by: Lars Hellemo <[email protected]>

Author: hellemo

.github/workflows/ci.yml

@@ -22,10 +22,10 @@ jobs:
           - version: '1'  # The latest point-release (Windows)
             os: windows-latest
             arch: x64
-          - version: '1.6'  # 1.6 LTS (64-bit Linux)
+          - version: '1.7'  # 1.7 for destructuring, not 1.6 LTS (64-bit Linux)
             os: ubuntu-latest
             arch: x64
-          - version: '1.6'  # 1.6 LTS (32-bit Linux)
+          - version: '1.7'  # 1.7 for destructuring, not 1.6 LTS (32-bit Linux)
             os: ubuntu-latest
             arch: x86
     steps:

.github/workflows/format_check.yml

@@ -19,7 +19,7 @@ jobs:
         run: |
           using Pkg
           # If you update the version, also update the style guide docs.
-          Pkg.add(PackageSpec(name="JuliaFormatter", version="1.0.8"))
+          Pkg.add(PackageSpec(name="JuliaFormatter", version="1.0.13"))
           using JuliaFormatter
           format("."; verbose = true)
           out = String(read(Cmd(`git diff`)))

NEWS.md

@@ -1,6 +1,15 @@
 TimeStructures release notes
 ===================================
 
+
+Version 0.7.0 (2022-10-25)
+--------------------------
+* Major cleanup and improved test coverage
+* Breaking changes:
+    - Remove `SparseVarArray' in favor of `IndexedVarArray`
+    - Remove custom macros and constructors in favor of exending standard JuMP macros
+    - Remove support for `DataFrame` and custom `Tables.jl` interface in favor of exending upstreamed Tables support in JuMP (will be added back when ready)
+
 Version 0.6.2 (2022-09-19)
 --------------------------
 * Add IndexedVarArray that checks for valid indices on insert and has improved performance

Project.toml

@@ -1,7 +1,7 @@
 name = "SparseVariables"
 uuid = "2749762c-80ed-4b14-8f33-f0736679b02b"
 authors = ["Truls Flatberg <[email protected]>", "Lars Hellemo <[email protected]>"]
-version = "0.6.2"
+version = "0.7.0"
 
 [deps]
 Dictionaries = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4"

README.md

@@ -3,8 +3,8 @@
 [![Build Status](https://github.com/hellemo/SparseVariables.jl/workflows/CI/badge.svg?branch=main)](https://github.com/hellemo/SparseVariables.jl/actions?query=workflow%3ACI)
 [![codecov](https://codecov.io/gh/hellemo/SparseVariables.jl/branch/main/graph/badge.svg?token=2LXGVU04YS)](https://codecov.io/gh/hellemo/SparseVariables.jl)
 
-This package contains routines for improved performance and easier handling of sparse data 
-and sparse arrays of optimizaton variables in JuMP. 
+Add container type(s) for improved performance and easier handling of sparse data 
+and sparse arrays of optimizaton variables in [JuMP](https://jump.dev/JuMP.jl/stable/). 
 
 Watch the JuliaCon/JuMP-dev 2022 lightning talk and check out the [notebook with examples and bencmarks]("docs/notebook_juliacon2022.jl"): 
 
@@ -29,7 +29,7 @@ const SV = SparseVariables
 m = Model()
 
 cars = ["ford", "bmw", "opel"]
-year = [2000, 2001, 2002, 2003]
+years = [2000, 2001, 2002, 2003]
 
 car_cost = SparseArray(Dict(
     ("ford", 2000) => 100,
@@ -38,91 +38,40 @@ car_cost = SparseArray(Dict(
     ("bmw", 2002) => 300
     ))
 
-# Variable defined for a given set of tuples
-@sparsevariable(m, y[car, year] for (car,year) in keys(car_cost))
-
-# Empty variable with 2 indices
-@sparsevariable(m, z[car, year])
 
+# Empty variables with 2 indices and allowed index values specified
+# by `car` and `year`, using `container=IndexedVarArray`
+@variable(m, y[car=cars, year=years]; container=IndexedVarArray)
+@variable(m, z[car=cars, year=years]; container=IndexedVarArray)
 # Dynamic creation of variables
+for (cr, yr) in keys(car_cost)
+    insertvar!(y, cr, yr)
+end
+
+# Inserting values not in the defined value sets errors:
 for c in ["opel", "tesla", "nikola"]
     insertvar!(z, c, 2002)
 end
 
+# Skip tests for allowed values for maximum performance.
+# Note that this will allow creating values outside the defined
+# sets, as long as the type is correct.
+for c in ["opel", "tesla", "nikola"]
+    unsafe_insertvar!(z, c, 2002)
+end
+
 # Inefficient iteration, but 0 contribution for non-existing variables
-@constraint(m, sum(y[c,i] + z[c,i] for c in cars, i in year) <= 300)
+@constraint(m, sum(y[c,i] + z[c,i] for c in cars, i in years) <= 300)
 
 # Slicing over selected indices
 @constraint(m, sum(y[:, 2000]) <= 300)
 
 # Efficient filtering using select syntax
-for i in year
+for i in years
     @constraint(m, sum(car_cost[c,i] * y[c,i] for (c,i) in SV.select(y, :, i)) <= 300)
 end
 
 # Filter using functions on indices
 @constraint(m, sum(z[endswith("a"), iseven]) >= 1)
 ```
 
-## IndexedVarArray
-
-Use IndexedVarArrays to check for valid indices and to warn against duplicate 
-indices, as well as improved performance:
-
-```julia
-    w = IndexedVarArray(m, "w", (car=cars, year=year))
-    m[:w] = w
-
-    for c in cars, y in year
-        insertvar!(w, c, y)
-    end
-```
-
-
-
-## Solution information
-
-SparseVariables.jl provides `SolutionTable` that supports the [Tables.jl](https://github.com/JuliaData/Tables.jl) interface, allowing 
-easy output of solution values to e.g. a `DataFrame` or a csv-file
-```julia
-using CSV
-using DataFrames
-using HiGHS
-
-# Solve m
-set_optimizer(m, HiGHS.Optimizer)
-optimize!(m)
-
-# Fetch solution
-tab = table(y)
-
-# Save to CSV
-CSV.write("result.csv", tab)
-
-# Convert to DataFrame
-df_y = dataframe(y)
-df_z = DataFrame(table(z))
-```
-The Tables interface is also implemented for `DenseAxisArray`, allowing the functionality to be used also for normal
-dense JuMP-variables. Since the container does not provide index names, these have to be given as explicit arguments:
-
-
-```julia
-# Add dense variable u
-@variable(m, u[cars, year])
-
-for c in cars, y in year
-    @constraint(m, u[c, y] <= 1)
-end
-
-# Solve
-set_optimizer(m, HiGHS.Optimizer)
-optimize!(m)
-
-# Read solution values for u
-tab = table(u, :u, :car, :year)
-df = DataFrame(tab)
-```
-
-Note that output to a DataFrame through the `dataframe` function is only possible if `DataFrames` is loaded
-before `SparseVariables`.

src/SparseVariables.jl

@@ -5,16 +5,12 @@ using JuMP
 using LinearAlgebra
 
 include("sparsearray.jl")
-include("sparsevararray.jl")
-include("macros.jl")
 include("dictionaries.jl")
 include("indexedarray.jl")
 include("tables.jl")
 
 export SparseArray
-export SparseVarArray
 export IndexedVarArray
-export @sparsevariable
 export insertvar!
 export unsafe_insertvar!
 

src/dictionaries.jl

@@ -1,7 +1,3 @@
-function variable_name(var::String, index)
-    return var * "[" * join(index, ", ") * "]"
-end
-
 """
     make_filter_fun(c, pos)
 
@@ -196,35 +192,13 @@ function select(dict::Dictionary, indices)
     return getindices(dict, select(keys(dict), indices))
 end
 select(dict, f::Function) = filter(f, dict)
-function kselect(sa::SparseVarArray, sh_pat::NamedTuple)
-    return select(keys(sa.data), sh_pat, get_index_names(sa))
-end
-function select(sa::SparseVarArray, sh_pat::NamedTuple)
-    return Dictionaries.getindices(sa, kselect(sa, sh_pat))
-end
 
 function select_test(dict, indices, cache)
     return cache ? _select_cached(dict, indices) :
            _select_gen(keys(dict), indices)
 end
 
-function _select_cached(sa, pat)
-    indices = Tuple(i for (i, v) in enumerate(pat) if v !== Colon())
-    vals = Tuple(v for v in pat if v !== Colon())
-
-    if !(indices in keys(sa.index_cache))
-        index = Dict()
-        for v in keys(sa)
-            vred = Tuple(val for (i, val) in enumerate(v) if i in indices)
-            if !(vred in keys(index))
-                index[vred] = []
-            end
-            push!(index[vred], v)
-        end
-        sa.index_cache[indices] = index
-    end
-    return get(sa.index_cache[indices], vals, [])
-end
+function _select_cached(sa, pat) end
 
 function permfromnames(names::NamedTuple, patnames)
     perm = (names[i] for i in patnames)

src/indexedarray.jl

@@ -10,67 +10,6 @@ struct IndexedVarArray{V<:AbstractVariableRef,N,T} <: AbstractSparseArray{V,N}
     index_cache::Vector{Dictionary}
 end
 
-function IndexedVarArray(
-    model::Model,
-    name::AbstractString,
-    index_names::NamedTuple{Ns,Ts};
-    lower_bound = 0,
-    kw_args...,
-) where {Ns,Ts}
-    T = Tuple{eltype.(fieldtypes(Ts))...}
-    N = length(fieldtypes(Ts))
-    dict = Dictionary{T,VariableRef}()
-    return model[Symbol(name)] = IndexedVarArray{VariableRef,N,T}(
-        (ix...) -> createvar(model, name, ix; lower_bound, kw_args...),
-        dict,
-        index_names,
-        Vector{Dictionary}(undef, 2^N),
-    )
-end
-
-function IndexedVarArray(
-    model::Model,
-    name::AbstractString,
-    index_names::NamedTuple{Ns,Ts},
-    indices::Vector{T};
-    lower_bound = 0,
-    kw_args...,
-) where {Ns,Ts,T}
-    @assert T == Tuple{eltype.(fieldtypes(Ts))...}
-    N = length(fieldtypes(Ts))
-    # TODO: Check if each index is valid
-    dict = Dictionary(
-        indices,
-        (createvar(model, name, k; lower_bound, kw_args...) for k in indices),
-    )
-    return model[Symbol(name)] = IndexedVarArray{VariableRef,N,T}(
-        (ix...) -> createvar(model, name, ix; lower_bound, kw_args...),
-        dict,
-        index_names,
-        Vector{Dictionary}(undef, 2^N),
-    )
-end
-
-function IndexedVarArray(
-    model::Model,
-    name::AbstractString,
-    index_names::NamedTuple{Ns,Ts},
-    indices::Dictionaries.Indices{T};
-    lower_bound = 0,
-    kw_args...,
-) where {Ns,Ts,T}
-    @assert T == Tuple{eltype.(fieldtypes(Ts))...}
-    N = length(fieldtypes(Ts))
-    return IndexedVarArray(
-        model,
-        name,
-        index_names,
-        collect(indices);
-        lower_bound,
-        kw_args...,
-    )
-end
-
 _data(sa::IndexedVarArray) = sa.data
 
 already_defined(var, index) = haskey(_data(var), index)
@@ -93,16 +32,11 @@ function clear_cache!(var)
 end
 
 """
-    insertvar!(var::IndexedVarArray{V,N,T}, index...; lower_bound = 0, kw_args...)
+    insertvar!(var::IndexedVarArray{V,N,T}, index...)
 
 Insert a new variable with the given index only after checking if keys are valid and not already defined.
 """
-function insertvar!(
-    var::IndexedVarArray{V,N,T},
-    index...;
-    lower_bound = 0,
-    kw_args...,
-) where {V,N,T}
+function insertvar!(var::IndexedVarArray{V,N,T}, index...) where {V,N,T}
     !valid_index(var, index) && throw(BoundsError(var, index))# "Not a valid index for $(var.name): $index"g
     already_defined(var, index) && error("$index already defined for array")
 
@@ -113,17 +47,12 @@ function insertvar!(
 end
 
 """
-    unsafe_insertvar!(var::indexedVarArray{V,N,T}, index...; lower_bound = 0, kw_args...)
+    unsafe_insertvar!(var::indexedVarArray{V,N,T}, index...)
 
 Insert a new variable with the given index withouth checking if the index is valid or 
  already assigned.
 """
-function unsafe_insertvar!(
-    var::IndexedVarArray{V,N,T},
-    index...;
-    lower_bound = 0,
-    kw_args...,
-) where {V,N,T}
+function unsafe_insertvar!(var::IndexedVarArray{V,N,T}, index...) where {V,N,T}
     return var[index] = var.f(index...)
 end
 
@@ -264,3 +193,10 @@ function Containers.container(f::Function, indices, D::Type{IndexedVarArray})
     index_vars = Symbol.("i$i" for i in 1:length(indices.prod.iterators))
     return Containers.container(f, indices, D, index_vars)
 end
+
+function Base.firstindex(sa::IndexedVarArray, d)
+    return first(sort(sa.index_names[d]))
+end
+function Base.lastindex(sa::IndexedVarArray, d)
+    return last(sort(sa.index_names[d]))
+end