diff --git a/.github/workflows/build-vehicle.yml b/.github/workflows/build-vehicle.yml index f77900be2..867af7a98 100644 --- a/.github/workflows/build-vehicle.yml +++ b/.github/workflows/build-vehicle.yml @@ -25,6 +25,7 @@ jobs: version: "3.10.2.1" project-file: "cabal.project.ghc-9.4.8" extra-args: "" + golden-test-args: "" include: # Build with GHC 9.8.1 - os: @@ -37,6 +38,8 @@ jobs: version: "3.10.2.1" project-file: "cabal.project.ghc-9.8.1" extra-args: "" + # The TestVerifier uses bash so only test on a single Linux version. + golden-test-args: "--test-option='--allowlist-externals=TestVerifier'" # Build with GHC 9.6.4 - os: name: Linux @@ -48,6 +51,7 @@ jobs: version: "3.10.2.1" project-file: "cabal.project.ghc-9.6.4" extra-args: "" + golden-test-args: "" # Build with GHC 9.2.8 - os: name: Linux @@ -59,6 +63,7 @@ jobs: version: "3.8.1.0" project-file: "cabal.project.ghc-9.2.8" extra-args: "" + golden-test-args: "" # Build with GHC 9.0.2 - os: name: Linux @@ -70,6 +75,7 @@ jobs: version: "3.8.1.0" project-file: "cabal.project.ghc-9.0.2" extra-args: "" + golden-test-args: "" # Build with GHC 8.10.7 - os: name: Linux @@ -81,6 +87,7 @@ jobs: version: "3.8.1.0" project-file: "cabal.project.ghc-8.10.7" extra-args: "" + golden-test-args: "" # Build with -fnothunks: - os: name: Linux @@ -92,6 +99,7 @@ jobs: version: "3.10.2.1" project-file: "cabal.project.nothunks.ghc-9.4.8" extra-args: "-fnothunks" + golden-test-args: "" # 20-12-2022: # This test is disabled because -fghc-debug requires two threads, which triggers #342. # Build with -fghc-debug: @@ -134,11 +142,10 @@ jobs: cabal-project-file: ${{ matrix.haskell.cabal.project-file }} cabal-project-freeze-file: ${{ matrix.haskell.cabal.project-file }}.freeze - - name: Test Vehicle + - name: Run Vehicle unit tests run: | cabal test \ vehicle:test:unit-tests \ - vehicle:test:golden-tests \ --test-show-details=always \ --test-option=--color=always \ --test-option=--num-threads=1 \ @@ -146,10 +153,24 @@ jobs: ${{ matrix.haskell.cabal.extra-args }} shell: sh + - name: Run Vehicle golden tests + run: | + cabal test \ + vehicle:test:golden-tests \ + --test-show-details=always \ + --test-option=--color=always \ + --test-option=--num-threads=1 \ + --test-option="--allowlist-externals=TestVerifier" \ + --project-file=${{ matrix.haskell.cabal.project-file }} \ + ${{ matrix.haskell.cabal.extra-args }} \ + ${{ matrix.haskell.cabal.golden-test-args }} + shell: sh + - name: Build Vehicle run: | mkdir -p bin - cabal install vehicle:exe:vehicle \ + cabal install \ + vehicle:exe:vehicle \ --overwrite-policy=always \ --install-method=copy \ --installdir=bin \ diff --git a/vehicle/tests/golden/compile/simple-fourierMotzkin/test.json b/vehicle/tests/golden/compile/simple-fourierMotzkin/test.json index 33c1ba213..8561c9f45 100644 --- a/vehicle/tests/golden/compile/simple-fourierMotzkin/test.json +++ b/vehicle/tests/golden/compile/simple-fourierMotzkin/test.json @@ -28,6 +28,7 @@ { "name": "TestVerify", "run": "vehicle verify -s spec.vcl -v TestVerifier --network f:fake.onnx --property underConstrainedVars --verifier-location=testVerifier --verifier-args='sat 'X_0=0.2,Y_0=3.0''", - "needs": ["spec.vcl", "fake.onnx", "../../../testVerifier"] + "needs": ["spec.vcl", "fake.onnx", "../../../testVerifier"], + "external": ["TestVerifier"] } ] diff --git a/vehicle/tests/golden/compile/windController/test.json b/vehicle/tests/golden/compile/windController/test.json index 2094b60dd..8d4fd4a49 100644 --- a/vehicle/tests/golden/compile/windController/test.json +++ b/vehicle/tests/golden/compile/windController/test.json @@ -34,7 +34,8 @@ { "name": "TestVerify", "run": "vehicle verify -s spec.vcl -v TestVerifier --network controller:controller.onnx --verifier-location=testVerifier --verifier-args='sat 'X_0=0.2,X_1=0.3,Y_0=3.0''", - "needs": ["spec.vcl", "controller.onnx", "../../../testVerifier"] + "needs": ["spec.vcl", "controller.onnx", "../../../testVerifier"], + "external": ["TestVerifier"] }, { "name": "Explicit",