testBuildFailure': init

doc/build-helpers/testers.chapter.md

@@ -255,6 +255,68 @@ runCommand "example" {
 
 :::
 
+## `testBuildFailure'` {#tester-testBuildFailurePrime}
+
+This tester wraps the functionality provided by [`testers.testBuildFailure`](#tester-testBuildFailure) to make writing checks easier by simplifying checking the exit code of the builder and asserting the existence of entries in the builder's log.
+Additionally, users may specify a script containing additional checks, accessing the result of applying `testers.testBuildFailure` through the variable `failed`.
+
+:::{.example #ex-testBuildFailurePrime-doc-example}
+
+# Check that a build fails, and verify the changes made during build
+
+```nix
+testers.testBuildFailure' {
+  drv = runCommand "doc-example" { } ''
+    echo ok-ish >"$out"
+    echo failing though
+    exit 3
+  '';
+  expectedBuilderExitCode = 3;
+  expectedBuilderLogEntries = [ "failing though" ];
+  script = ''
+    grep --silent -F 'ok-ish' "$failed/result"
+    touch "$out"
+  '';
+}
+```
+
+:::
+
+
+### Inputs {#tester-testBuildFailurePrime-inputs}
+
+`drv` (derivation)
+
+: The failing derivation to wrap with `testBuildFailure`.
+
+`name` (string, optional)
+
+: The name of the test.
+  When not provided, this value defaults to `testBuildFailure-${(testers.testBuildFailure drv).name}`.
+
+`expectedBuilderExitCode` (integer, optional)
+
+: The expected exit code of the builder of `drv`.
+  When not provided, this value defaults to `1`.
+
+`expectedBuilderLogEntries` (array of string-like values, optional)
+
+: A list of string-like values which must be found in the builder's log by exact match.
+  When not provided, this value defaults to `[ ]`.
+
+  NOTE: Patterns and regular expressions are not supported.
+
+`script` (string, optional)
+
+: A string containing additional checks to run.
+  When not provided, this value defaults to `""`.
+  The result of `testers.testBuildFailure drv` is available through the variable `failed`.
+  As an example, the builder's log is at `"$failed/testBuildFailure.log"`.
+
+### Return value {#tester-testBuildFailurePrime-return}
+
+The tester produces an empty output and only succeeds when the checks using `expectedBuilderExitCode`, `expectedBuilderLogEntries`, and `script` succeed.
+
 ## `testEqualContents` {#tester-testEqualContents}
 
 Check that two paths have the same contents.

doc/redirects.json

@@ -8,6 +8,9 @@
   "ex-build-helpers-extendMkDerivation": [
     "index.html#ex-build-helpers-extendMkDerivation"
   ],
+  "ex-testBuildFailurePrime-doc-example": [
+    "index.html#ex-testBuildFailurePrime-doc-example"
+  ],
   "neovim": [
     "index.html#neovim"
   ],
@@ -332,6 +335,15 @@
   "footnote-stdenv-find-inputs-location.__back.0": [
     "index.html#footnote-stdenv-find-inputs-location.__back.0"
   ],
+  "tester-testBuildFailurePrime": [
+    "index.html#tester-testBuildFailurePrime"
+  ],
+  "tester-testBuildFailurePrime-inputs": [
+    "index.html#tester-testBuildFailurePrime-inputs"
+  ],
+  "tester-testBuildFailurePrime-return": [
+    "index.html#tester-testBuildFailurePrime-return"
+  ],
   "variables-specifying-dependencies": [
     "index.html#variables-specifying-dependencies"
   ],

pkgs/build-support/testers/default.nix

@@ -34,6 +34,72 @@
     ] ++ orig.args or ["-e" ../../stdenv/generic/source-stdenv.sh (orig.builder or ../../stdenv/generic/default-builder.sh)];
   });
 
+  # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime
+  # or doc/build-helpers/testers.chapter.md
+  testBuildFailure' =
+    let
+      mkBuildCommand =
+        script:
+        ''
+          if [[ -n ''${expectedBuilderExitCode:-} ]]; then
+            nixLog "checking original builder exit code"
+            builderExitCode=$(<"$failed/testBuildFailure.exit")
+            if ((expectedBuilderExitCode == builderExitCode)); then
+              nixLog "original builder exit code matches expected value of $expectedBuilderExitCode"
+            else
+              nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode"
+              exit 1
+            fi
+            unset builderExitCode
+          fi
+
+          if ((''${#expectedBuilderLogEntries[@]})); then
+            nixLog "checking original builder log"
+            builderLogEntries="$(<"$failed/testBuildFailure.log")"
+            shouldExit=0
+            for expectedBuilderLogEntry in "''${expectedBuilderLogEntries[@]}"; do
+              if [[ ''${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then
+                nixLog "original builder log contains ''${expectedBuilderLogEntry@Q}"
+              else
+                nixErrorLog "original builder log does not contain ''${expectedBuilderLogEntry@Q}"
+                shouldExit=1
+              fi
+            done
+            unset builderLogEntries
+            ((shouldExit)) && exit 1
+            unset shouldExit
+          fi
+        ''
+        + lib.optionalString (script != "") ''
+          nixLog "running additional checks from user-provided script"
+          ${script}
+        ''
+        + ''
+          touch "$out"
+        '';
+      final =
+        {
+          drv,
+          name ? null,
+          expectedBuilderExitCode ? 1, # NOTE: Should be an integer.
+          expectedBuilderLogEntries ? [ ], # NOTE: Should be an array of string-coercible values. TODO: Only checks for inclusion, not order!
+          script ? "", # Succeed by default if checks pass.
+        }:
+        (runCommand name {
+          __structuredAttrs = true;
+          strictDeps = true;
+          failed = testers.testBuildFailure drv;
+          inherit expectedBuilderExitCode expectedBuilderLogEntries;
+        } (mkBuildCommand script)).overrideAttrs
+          (
+            finalAttrs: _: {
+              # Fix name so the default value uses whatever failed ends up as.
+              name = if name != null then name else "testBuildFailure-${finalAttrs.failed.name}";
+            }
+          );
+    in
+    lib.makeOverridable final;
+
   # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation
   # or doc/build-helpers/testers.chapter.md
   testEqualDerivation = callPackage ./test-equal-derivation.nix { };

pkgs/build-support/testers/test/default.nix

@@ -220,6 +220,120 @@ lib.recurseIntoAttrs {
     sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects;
   };
 
+  testBuildFailure' = lib.recurseIntoAttrs rec {
+    # NOTE: This example is used in the docs.
+    # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime
+    # or doc/build-helpers/testers.chapter.md
+    doc-example = testers.testBuildFailure' {
+      drv = runCommand "doc-example" { } ''
+        echo ok-ish >"$out"
+        echo failing though
+        exit 3
+      '';
+      expectedBuilderExitCode = 3;
+      expectedBuilderLogEntries = [ "failing though" ];
+      script = ''
+        grep --silent -F 'ok-ish' "$failed/result"
+        touch "$out"
+      '';
+    };
+
+    happy = testers.testBuildFailure' {
+      drv = runCommand "happy" { } ''
+        echo ok-ish >$out
+
+        echo failing though
+        echo also stderr 1>&2
+        echo 'line\nwith-\bbackslashes'
+        printf "incomplete line - no newline"
+
+        exit 3
+      '';
+      expectedBuilderExitCode = 3;
+      expectedBuilderLogEntries = [
+        "failing though"
+        "also stderr"
+        ''line\nwith-\bbackslashes''
+        "incomplete line - no newline"
+      ];
+      script = ''
+        grep --silent -F 'ok-ish' "$failed/result"
+        touch "$out"
+      '';
+    };
+
+    happyStructuredAttrs = overrideStructuredAttrs true happy;
+
+    helloDoesNotFail = testers.testBuildFailure' {
+      drv = testers.testBuildFailure hello;
+      expectedBuilderLogEntries = [
+        "testBuildFailure: The builder did not fail, but a failure was expected"
+      ];
+    };
+
+    multiOutput = testers.testBuildFailure' {
+      drv =
+        runCommand "multiOutput"
+          {
+            # dev will be the default output
+            outputs = [
+              "dev"
+              "doc"
+              "out"
+            ];
+          }
+          ''
+            echo i am failing
+            exit 1
+          '';
+      expectedBuilderLogEntries = [
+        "i am failing"
+      ];
+      script = ''
+        # Checking our note that dev is the default output
+        echo $failed/_ | grep -- '-dev/_' >/dev/null
+        echo 'All good.'
+        touch $out
+      '';
+    };
+
+    multiOutputStructuredAttrs = overrideStructuredAttrs true multiOutput;
+
+    sideEffects = testers.testBuildFailure' {
+      drv = stdenvNoCC.mkDerivation {
+        name = "fail-with-side-effects";
+        src = emptyDirectory;
+
+        postHook = ''
+          echo touching side-effect...
+          # Assert that the side-effect doesn't exist yet...
+          # We're checking that this hook isn't run by expect-failure.sh
+          if [[ -e side-effect ]]; then
+            echo "side-effect already exists"
+            exit 1
+          fi
+          touch side-effect
+        '';
+
+        buildPhase = ''
+          echo i am failing
+          exit 1
+        '';
+      };
+      expectedBuilderLogEntries = [
+        "touching side-effect..."
+        "i am failing"
+      ];
+      script = ''
+        [[ ! -e side-effect ]]
+
+        touch $out
+      '';
+    };
+
+    sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects;
+  };
+
   testEqualContents = lib.recurseIntoAttrs {
     equalDir = testers.testEqualContents {
       assertion = "The same directory contents at different paths are recognized as equal";