From 79e6c0e2f6a214a27447dab9ee110cc2c7bd7aa6 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 19 Feb 2025 15:50:32 -0800 Subject: [PATCH] testBuildFailure': init --- doc/build-helpers/testers.chapter.md | 62 +++++++++++ doc/redirects.json | 12 +++ pkgs/build-support/testers/default.nix | 66 ++++++++++++ pkgs/build-support/testers/test/default.nix | 114 ++++++++++++++++++++ 4 files changed, 254 insertions(+) diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index e0eb0cd1a5de68..3aa2e872b1a45d 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -255,6 +255,68 @@ runCommand "example" { ::: +## `testBuildFailure'` {#tester-testBuildFailurePrime} + +This tester wraps the functionality provided by [`testers.testBuildFailure`](#tester-testBuildFailure) to make writing checks easier by simplifying checking the exit code of the builder and asserting the existence of entries in the builder's log. +Additionally, users may specify a script containing additional checks, accessing the result of applying `testers.testBuildFailure` through the variable `failed`. + +:::{.example #ex-testBuildFailurePrime-doc-example} + +# Check that a build fails, and verify the changes made during build + +```nix +testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + touch "$out" + ''; +} +``` + +::: + + +### Inputs {#tester-testBuildFailurePrime-inputs} + +`drv` (derivation) + +: The failing derivation to wrap with `testBuildFailure`. + +`name` (string, optional) + +: The name of the test. + When not provided, this value defaults to `testBuildFailure-${(testers.testBuildFailure drv).name}`. + +`expectedBuilderExitCode` (integer, optional) + +: The expected exit code of the builder of `drv`. + When not provided, this value defaults to `1`. + +`expectedBuilderLogEntries` (array of string-like values, optional) + +: A list of string-like values which must be found in the builder's log by exact match. + When not provided, this value defaults to `[ ]`. + + NOTE: Patterns and regular expressions are not supported. + +`script` (string, optional) + +: A string containing additional checks to run. + When not provided, this value defaults to `""`. + The result of `testers.testBuildFailure drv` is available through the variable `failed`. + As an example, the builder's log is at `"$failed/testBuildFailure.log"`. + +### Return value {#tester-testBuildFailurePrime-return} + +The tester produces an empty output and only succeeds when the checks using `expectedBuilderExitCode`, `expectedBuilderLogEntries`, and `script` succeed. + ## `testEqualContents` {#tester-testEqualContents} Check that two paths have the same contents. diff --git a/doc/redirects.json b/doc/redirects.json index ddf3626bbd3409..fd3388f3b3b425 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -8,6 +8,9 @@ "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], + "ex-testBuildFailurePrime-doc-example": [ + "index.html#ex-testBuildFailurePrime-doc-example" + ], "neovim": [ "index.html#neovim" ], @@ -332,6 +335,15 @@ "footnote-stdenv-find-inputs-location.__back.0": [ "index.html#footnote-stdenv-find-inputs-location.__back.0" ], + "tester-testBuildFailurePrime": [ + "index.html#tester-testBuildFailurePrime" + ], + "tester-testBuildFailurePrime-inputs": [ + "index.html#tester-testBuildFailurePrime-inputs" + ], + "tester-testBuildFailurePrime-return": [ + "index.html#tester-testBuildFailurePrime-return" + ], "variables-specifying-dependencies": [ "index.html#variables-specifying-dependencies" ], diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 0393895d53affa..ec3ddae2e29a86 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -34,6 +34,72 @@ ] ++ orig.args or ["-e" ../../stdenv/generic/source-stdenv.sh (orig.builder or ../../stdenv/generic/default-builder.sh)]; }); + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + testBuildFailure' = + let + mkBuildCommand = + script: + '' + if [[ -n ''${expectedBuilderExitCode:-} ]]; then + nixLog "checking original builder exit code" + builderExitCode=$(<"$failed/testBuildFailure.exit") + if ((expectedBuilderExitCode == builderExitCode)); then + nixLog "original builder exit code matches expected value of $expectedBuilderExitCode" + else + nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode" + exit 1 + fi + unset builderExitCode + fi + + if ((''${#expectedBuilderLogEntries[@]})); then + nixLog "checking original builder log" + builderLogEntries="$(<"$failed/testBuildFailure.log")" + shouldExit=0 + for expectedBuilderLogEntry in "''${expectedBuilderLogEntries[@]}"; do + if [[ ''${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then + nixLog "original builder log contains ''${expectedBuilderLogEntry@Q}" + else + nixErrorLog "original builder log does not contain ''${expectedBuilderLogEntry@Q}" + shouldExit=1 + fi + done + unset builderLogEntries + ((shouldExit)) && exit 1 + unset shouldExit + fi + '' + + lib.optionalString (script != "") '' + nixLog "running additional checks from user-provided script" + ${script} + '' + + '' + touch "$out" + ''; + final = + { + drv, + name ? null, + expectedBuilderExitCode ? 1, # NOTE: Should be an integer. + expectedBuilderLogEntries ? [ ], # NOTE: Should be an array of string-coercible values. TODO: Only checks for inclusion, not order! + script ? "", # Succeed by default if checks pass. + }: + (runCommand name { + __structuredAttrs = true; + strictDeps = true; + failed = testers.testBuildFailure drv; + inherit expectedBuilderExitCode expectedBuilderLogEntries; + } (mkBuildCommand script)).overrideAttrs + ( + finalAttrs: _: { + # Fix name so the default value uses whatever failed ends up as. + name = if name != null then name else "testBuildFailure-${finalAttrs.failed.name}"; + } + ); + in + lib.makeOverridable final; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation # or doc/build-helpers/testers.chapter.md testEqualDerivation = callPackage ./test-equal-derivation.nix { }; diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index b0f2b4c1d391a6..ac05f95f54a985 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -220,6 +220,120 @@ lib.recurseIntoAttrs { sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; }; + testBuildFailure' = lib.recurseIntoAttrs rec { + # NOTE: This example is used in the docs. + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + doc-example = testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + touch "$out" + ''; + }; + + happy = testers.testBuildFailure' { + drv = runCommand "happy" { } '' + echo ok-ish >$out + + echo failing though + echo also stderr 1>&2 + echo 'line\nwith-\bbackslashes' + printf "incomplete line - no newline" + + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ + "failing though" + "also stderr" + ''line\nwith-\bbackslashes'' + "incomplete line - no newline" + ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + touch "$out" + ''; + }; + + happyStructuredAttrs = overrideStructuredAttrs true happy; + + helloDoesNotFail = testers.testBuildFailure' { + drv = testers.testBuildFailure hello; + expectedBuilderLogEntries = [ + "testBuildFailure: The builder did not fail, but a failure was expected" + ]; + }; + + multiOutput = testers.testBuildFailure' { + drv = + runCommand "multiOutput" + { + # dev will be the default output + outputs = [ + "dev" + "doc" + "out" + ]; + } + '' + echo i am failing + exit 1 + ''; + expectedBuilderLogEntries = [ + "i am failing" + ]; + script = '' + # Checking our note that dev is the default output + echo $failed/_ | grep -- '-dev/_' >/dev/null + echo 'All good.' + touch $out + ''; + }; + + multiOutputStructuredAttrs = overrideStructuredAttrs true multiOutput; + + sideEffects = testers.testBuildFailure' { + drv = stdenvNoCC.mkDerivation { + name = "fail-with-side-effects"; + src = emptyDirectory; + + postHook = '' + echo touching side-effect... + # Assert that the side-effect doesn't exist yet... + # We're checking that this hook isn't run by expect-failure.sh + if [[ -e side-effect ]]; then + echo "side-effect already exists" + exit 1 + fi + touch side-effect + ''; + + buildPhase = '' + echo i am failing + exit 1 + ''; + }; + expectedBuilderLogEntries = [ + "touching side-effect..." + "i am failing" + ]; + script = '' + [[ ! -e side-effect ]] + + touch $out + ''; + }; + + sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; + }; + testEqualContents = lib.recurseIntoAttrs { equalDir = testers.testEqualContents { assertion = "The same directory contents at different paths are recognized as equal";