Merge pull request #76 from SCIP-Interfaces/rs/clang

Rewrite SCIP wrapper with Clang.jl to support a MIP solver through MOI

Author: rschwarz

.gitignore

@@ -3,9 +3,4 @@
 *.jl.*.cov
 *.jl.mem
 deps/deps.jl
-deps/downloads/
-deps/src/
-deps/usr/
 deps/build.log
-/test/test.lp
-/test/test.mps

.travis.yml

@@ -1,23 +1,27 @@
-# Documentation: http://docs.travis-ci.com/user/languages/julia/
+## Documentation: http://docs.travis-ci.com/user/languages/julia/
 language: julia
-os:
-  - linux
+os: linux
+dist: xenial
+sudo: true
 julia:
-  - 0.7
   - 1.0
-dist: trusty
-sudo: required
-services:
-  - docker
-before_install:
-  - echo "**** pulling Docker image"
-  - docker pull leethargo/scip-julia
+  - nightly
 notifications:
   email: false
+git:
+  depth: 99999999
+matrix:
+  allow_failures:
+    - julia: nightly
+before_install:
+  - sudo apt-get install libblas-dev liblapack-dev gfortran
+  - export VERSION=6.0.0
+  - wget http://scip.zib.de/download/release/SCIPOptSuite-$VERSION-Linux.deb
+  - sudo dpkg -i SCIPOptSuite-$VERSION-Linux.deb
 script:
-  - echo "**** running Docker"
-  - docker run --env-file travis_docker_env.list -t -a STDOUT -a STDIN -a STDERR -v $PWD:/mnt leethargo/scip-julia /mnt/travis_docker_test_script.sh $TRAVIS_JULIA_VERSION
+  - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("SCIP"); Pkg.test("SCIP"; coverage=true)'
 after_success:
-  # push coverage results to Coveralls, .cov files were copied back by script above
-  - echo "**** submitting coverage information"
-  - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(process_folder()); Codecov.submit(Codecov.process_folder())'
+  # push coverage results to Coveralls
+  - julia -e 'using Pkg; cd(Pkg.dir("SCIP")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'
+  # push coverage results to Codecov
+  - julia -e 'using Pkg; cd(Pkg.dir("SCIP")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Robert Schwarz
+Copyright (c) 2018 Felipe Serrano, Miles Lubin, Robert Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,64 +1,91 @@
 # SCIP.jl
+
 Julia interface to [SCIP](http://scip.zib.de) solver.
 
 [![Build Status](https://travis-ci.org/SCIP-Interfaces/SCIP.jl.svg?branch=master)](https://travis-ci.org/SCIP-Interfaces/SCIP.jl)
 [![Coverage Status](https://coveralls.io/repos/github/SCIP-Interfaces/SCIP.jl/badge.svg?branch=master)](https://coveralls.io/github/SCIP-Interfaces/SCIP.jl?branch=master)
 [![codecov](https://codecov.io/gh/SCIP-Interfaces/SCIP.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/SCIP-Interfaces/SCIP.jl)
 
-## Related Projects
-
-- [SCIP](http://scip.zib.de): actual solver (implemented in C) that is wrapped
-  for Julia.
-- [CSIP](https://github.com/SCIP-Interfaces/CSIP): restricted and simplified C
-  interface to SCIP which our wrapper is based on.
-- [SCIP.jl](https://github.com/ryanjoneil/SCIP.jl): previous attempt to
-  interface SCIP from Julia, using autogenerated wrapper code for all public
-  functions.
-- [MathProgBase](https://github.com/JuliaOpt/MathProgBase.jl): We aim to
-  implement MPB's abstract solver interfaces, so that one can use SCIP.jl
-  through [JuMP](https://github.com/JuliaOpt/JuMP.jl). For now, the
-  `LinearQuadraticModel` interface is implemented, supporting lazy constraint
-  and heuristic callbacks.
-
-## Installation
-
-**Note**: These instructions are meant for and only tested with GNU/Linux. OS X used to work, 
-but there is an issue (#46) since the update to SCIP 4.0.0.
-
-Follow the steps below to get SCIP.jl working. Unfortunately, these steps can not be automated as part of `Pkg.build("SCIP")`, because the academic license of SCIP does not allow distribution of the source code without tracking the download metadata. See the [license](http://scip.zib.de/academic.txt) for details.
-
-1.The SCIP.jl package requires [SCIP](http://scip.zib.de/) to be installed in a recent version (e.g. 6.0.0).
-[Download](http://scip.zib.de) the SCIP Optimization Suite.
-```
-tar xzf scipoptsuite-6.0.0.tgz
-```
-
-2.Choose an installation path and set the **environment variable `SCIPOPTDIR`** to point there.
-```
-export SCIPOPTDIR=`my/install/dir`
-```
-
-3.Build and install the shared library with
-```
-mkdir build
-cd build
-cmake -DCMAKE_INSTALL_PREFIX=$SCIPOPTDIR ..
-make
-make install
-```
-
-4.This package is registered in `METADATA.jl` and can be installed in Julia with
-```
-Pkg.add("SCIP")
-```
-
-## Setting Parameters
-
-SCIP has a [long list of parameters](http://scip.zib.de/doc/html/PARAMETERS.php)
-that can all be set through SCIP.jl, by passing them to the constructor of
-`SCIPSolver`. To set a value `val` to a parameter `name`, pass the two
-parameters `(name, val)`. For example, let's set two parameters, to disable
-output and increase the gap limit to 0.05:
-```
-solver = SCIPSolver("display/verblevel", 0, "limits/gap", 0.05)
-```
+## Update (December 2018)
+
+We have completely rewritten the interface from scratch, using
+[Clang.jl](https://github.com/ihnorton/Clang.jl) to generate wrappers based on
+the headers of the SCIP library.
+The goal is to support [JuMP](https://github.com/JuliaOpt/JuMP.jl) (from version
+0.19 on) through
+[MathOptInterface](https://github.com/JuliaOpt/MathOptInterface.jl).
+
+Currently, we support LP and MIP problems only.
+This means we still have a feature loss in the areas as nonlinear constraints as
+well as supported callbacks compared to previous versions (see below).
+
+## Getting Started
+
+To use SCIP.jl, you will need [SCIP](http://scip.zib.de) installed on your
+system. [SCIP's license](https://scip.zib.de/index.php#license) does not allow
+(automatic) redistribution, so please
+[download](https://scip.zib.de/index.php#download) and install it yourself.
+
+Only Linux is tested and officially supported. Contributions to supporting other
+operating systems are welcome.
+
+We recommend using one of the provided installers, e.g.,
+`SCIPOptSuite-6.0.0-Linux.deb` for systems based on Debian. Adding the SCIP.jl
+package should then work out of the box:
+
+    pkg> add SCIP
+
+If you [build SCIP from source](https://scip.zib.de/doc-6.0.0/html/CMAKE.php)
+you should set the environment variable `SCIPOPTDIR` to point the the
+**installation path**. That is, `$SCIPOPTDIR/lib/libscip.so` should exist.
+
+## Design Considerations
+
+**Wrapper of Public API**: All of SCIP's public API methods are wrapped and
+available within the `SCIP` package. This includes the `scip_*.h` and `pub_*.h`
+headers that are collected in `scip.h`, as well as all default constraint
+handlers (`cons_*.h`.) But the wrapped functions do not transform any data
+structures and work on the *raw* pointers (e.g. `SCIP*` in C, `Ptr{SCIP_}` in
+Julia). Convenience wrapper functions based on Julia types are added as needed.
+
+**Memory Management**: Programming with SCIP requires dealing with variable and
+constraints objects that use [reference
+counting](https://scip.zib.de/doc-6.0.0/html/OBJ.php) for memory management.
+SCIP.jl provides a wrapper type `ManagedSCIP` that collects lists of `SCIP_VAR*`
+and `SCIP_CONS*` under the hood, and releases all reference when it is garbage
+collected itself (via `finalize`). When adding a variable (`add_variable`) or a
+constraint (`add_linear_constraint`), an integer index is returned. This index
+can be used to retrieve the `SCIP_VAR*` or `SCIP_CONS*` pointer via `get_var`
+and `get_cons` respectively.
+
+`ManagedSCIP` does not currently support deletion of variables or constraints.
+
+**Supported Features for MathOptInterface**: We aim at exposing many of SCIP's
+features through MathOptInterface. However, the focus is on keeping the wrapper
+simple and avoiding duplicate storage of model data.
+
+As a consequence, we do not currently support some features such as retrieving
+constraints by name (`SingleVariable`-set constraints are not stored as SCIP
+constraints explicitly).
+
+Support for more constraint types (quadratic/SOC, SOS1/2, nonlinear expression)
+is planned, but SCIP itself only supports affine objective functions, so we will
+stick with that. More general objective functions could be implented via a
+[bridge](https://github.com/JuliaOpt/MathOptInterface.jl/issues/529).
+
+## Old Interface Implementation
+
+A previous implementation of SCIP.jl supported
+[JuMP](https://github.com/JuliaOpt/JuMP.jl) (up to version 0.18) through
+[MathProgBase](https://github.com/JuliaOpt/MathOptInterface.jl). It did not
+interface SCIP directly, but went through
+[CSIP](https://github.com/SCIP-Interfaces/CSIP), a simplified C wrapper.
+
+Back then, the interface support MINLP problems as well as solver-indepentent
+callbacks for lazy constraints and heuristics.
+
+To use that version, you need to pin the version of SCIP.jl to `v0.6.1` (the
+last release before the rewrite):
+
+    pkg> add [email protected]
+    pkg> pin SCIP

REQUIRE

@@ -1,3 +1,2 @@
-julia 0.7
-BinDeps
-MathProgBase 0.5 0.8
+julia 1.0
+MathOptInterface 0.8

benchmark/sparse_linprog.jl

@@ -1,54 +1,41 @@
-using BinDeps
-
 using Libdl
 
-include("csip_version.jl")
-
-CSIP_URL = "https://github.com/SCIP-Interfaces/CSIP/archive/v$(CSIP_VERSION).zip"
-CSIP_LIB = "libcsip"
-CSIP_UNPACKED = "CSIP-$(CSIP_VERSION)"
-
-@BinDeps.setup
-
-@assert haskey(ENV, "SCIPOPTDIR") "Environment variable `SCIPOPTDIR` not set!"
-
-function validate_csip(name, handle)
-    csip_version = ccall(Libdl.dlsym(handle, :CSIPgetVersion), Cint, ())
-    csip_version == csip_required_int
+depsfile = joinpath(dirname(@__FILE__), "deps.jl")
+if isfile(depsfile)
+    rm(depsfile)
 end
 
-csipdep = library_dependency(CSIP_LIB, validate=validate_csip)
-
-provides(Sources, Dict(URI(CSIP_URL) => csipdep),
-                 unpacked_dir=CSIP_UNPACKED)
-
-src_dir = joinpath(BinDeps.srcdir(csipdep), CSIP_UNPACKED)
-scipopt_dir = ENV["SCIPOPTDIR"]
-
-provides(SimpleBuild,
-   (@build_steps begin
-        GetSources(csipdep)
-        @build_steps begin
-            ChangeDirectory(joinpath(BinDeps.srcdir(csipdep), CSIP_UNPACKED))
-            `make`
-            `gcc -std=c99 -Wall -pedantic -I$(src_dir)/lib/include -I$(src_dir)/include -c $(src_dir)/src/csip.c -L$(src_dir)/lib -Wl,-rpath,$(scipopt_dir)/lib/ $(scipopt_dir)/lib/libscipopt.dylib -fPIC -o $(src_dir)/src/csip.o`
-            `   gcc -std=c99 -Wall -pedantic $(src_dir)/src/csip.o -L$(src_dir)/lib -Wl,-rpath,$(scipopt_dir)/lib/ $(scipopt_dir)/lib/libscipopt.dylib -fPIC -shared -dynamiclib -o $(src_dir)/lib/libcsip.dylib`
-            `mkdir -p $(libdir(csipdep))`
-            `install lib/libcsip.dylib $(libdir(csipdep))`
-        end
-   end), csipdep, os = :Darwin)
+function write_depsfile(path)
+    f = open(depsfile, "w")
+    println(f, "const libscip = \"$path\"")
+    close(f)
+end
 
+libname = "libscip.so"
+paths_to_try = []
 
-provides(SimpleBuild,
-    (@build_steps begin
-         GetSources(csipdep)
-         @build_steps begin
-             ChangeDirectory(joinpath(BinDeps.srcdir(csipdep), CSIP_UNPACKED))
-             `make`
-             `mkdir -p $(libdir(csipdep))`
-             `install lib/libcsip.so $(libdir(csipdep))`
-         end
-    end), csipdep, os = :Unix)
+# prefer environment variable
+if haskey(ENV, "SCIPOPTDIR")
+    if Sys.islinux()
+        push!(paths_to_try, joinpath(ENV["SCIPOPTDIR"], "lib", libname))
+    end
+end
 
+# but also try library path
+push!(paths_to_try, libname)
+
+found = false
+for l in paths_to_try
+    d = Libdl.dlopen_e(l)
+    if d != C_NULL
+        global found = true
+        write_depsfile(l)
+        break
+    end
+end
 
-@BinDeps.install Dict(:libcsip => :libcsip)
+if !found && !haskey(ENV, "SCIP_JL_SKIP_LIB_CHECK")
+    error("Unable to locate SCIP installation. " *
+          "Note that this must be downloaded separately from scip.zib.de. " *
+          "Please set the environment variable SCIPOPTDIR to SCIP's installation path.")
+end

deps/build.jl

@@ -0,0 +1,31 @@
+using Clang
+
+HEADER_BASE = "/usr/include" # using system-wide installation of SCIP
+all_headers = readdir(joinpath(HEADER_BASE, "scip"))
+scip_headers = vcat(
+    filter(h -> startswith(h, "type_"), all_headers),
+    filter(h -> startswith(h, "pub_"), all_headers),
+    filter(h -> startswith(h, "scip_"), all_headers),
+    "scipdefplugins.h",
+    filter(h -> startswith(h, "cons_"), all_headers),
+)
+lpi_headers = ["type_lpi.h"]
+nlpi_headers = ["type_expr.h", "type_nlpi.h"]
+
+headers = vcat(
+    [joinpath(HEADER_BASE, "scip", h) for h in scip_headers],
+    [joinpath(HEADER_BASE, "lpi", h) for h in lpi_headers],
+    [joinpath(HEADER_BASE, "nlpi", h) for h in nlpi_headers],
+)
+
+context = Clang.init(
+    headers=headers,
+    common_file="commons.jl",
+    output_dir="../src/wrapper",
+    clang_includes=vcat(HEADER_BASE, LLVM_INCLUDE),
+    clang_args = ["-I", HEADER_BASE],
+    clang_diagnostics=true,
+    header_wrapped=(header, cursorname) -> header == cursorname,
+    header_library=header_name -> "libscip"
+)
+Clang.run(context)