flakes: ensure that flake packages don't throw during construction

Our `tools` construction returns `null` for packages that are missing (for whatever reason).
While this works okay for our internal module system, external consumers of the packages output sometimes make the assumption that these are all valid derivations.

At some point we added a filter that would remove the nulls, however this introduced even worse issues.
To filter out the nulls, we need to evaluate each tool.
But not all tools can be evaluated: some are stubs that throws (useful) errors.
So now we're forced to supress removal notices and other messages to be able to evaluate the tool list.

This commit refactors tools to return placeholder derivations.
These can be safely filtered out if needed.

Author: sandydoo

.github/workflows/ci.yaml

@@ -39,7 +39,8 @@ jobs:
     - run: rm -rf /opt&
 
     - name: Check nixpkgs-unstable
-      run: nix flake check -L --show-trace
+      working-directory: ./dev
+      run: nix flake check -L --show-trace --no-write-lockfile
 
     - run: nix eval .#lib.x86_64-linux.run --show-trace
 
@@ -58,4 +59,5 @@ jobs:
     - run: rm -rf /opt&
 
     - name: Check nixpkgs-stable
+      working-directory: ./dev
       run: nix flake check -L --show-trace --override-input nixpkgs github:NixOS/nixpkgs/nixos-25.05

dev/flake.nix

@@ -0,0 +1,15 @@
+{
+  description = "A very basic flake";
+
+  inputs = {
+    git-hooks.url = "path:..";
+  };
+
+  outputs =
+    { git-hooks
+    , ...
+    }:
+    {
+      inherit (git-hooks) legacyPackages checks;
+    };
+}

flake.nix

@@ -40,6 +40,11 @@
         '';
       };
 
+      legacyPackages = self.packages;
+
+      # The set of tools exposed by git-hooks.
+      # Each entry is guaranteed to be a derivation, but broken packages are not filtered out.
+      # `nix flake check` will likely not work.
       packages = forAllSystems ({ exposed, ... }: exposed.packages // {
         default = exposed.packages.pre-commit;
       });
@@ -50,7 +55,7 @@
         };
       });
 
-      checks = forAllSystems ({ exposed, ... }: lib.filterAttrs (k: v: v != null) exposed.checks);
+      checks = forAllSystems ({ exposed, ... }: exposed.checks);
 
       lib = forAllSystems ({ exposed, ... }: { inherit (exposed) run; });
 

modules/hooks.nix

@@ -2618,7 +2618,7 @@ in
             # need version >= 0.4.0 for the --from-stdin flag
             toolVersionCheck = lib.versionAtLeast convco.version "0.4.0";
           in
-          lib.throwIf (convco == null || !toolVersionCheck) "The version of Nixpkgs used by git-hooks.nix does not have the `convco` package (>=0.4.0). Please use a more recent version of Nixpkgs."
+          lib.throwIfNot toolVersionCheck "The version of Nixpkgs used by git-hooks.nix does not have the `convco` package (>=0.4.0). Please use a more recent version of Nixpkgs."
             builtins.toString
             script;
         stages = [ "commit-msg" ];
@@ -3089,9 +3089,7 @@ lib.escapeShellArgs (lib.concatMap (ext: [ "--ghc-opt" "-X${ext}" ]) hooks.fourm
                 "$PRE_COMMIT_COMMIT_MSG_SOURCE" --commit-msg-file "$1"
             '';
           in
-          lib.throwIf (hooks.gptcommit.package == null) "The version of Nixpkgs used by git-hooks.nix does not have the `gptcommit` package. Please use a more recent version of Nixpkgs."
-            toString
-            script;
+          toString script;
         stages = [ "prepare-commit-msg" ];
       };
       hadolint =
@@ -3110,13 +3108,7 @@ lib.escapeShellArgs (lib.concatMap (ext: [ "--ghc-opt" "-X${ext}" ]) hooks.fourm
           ## https://github.com/Frama-C/headache/blob/master/config_builtin.txt
           files = "(\\.ml[ily]?$)|(\\.fmli?$)|(\\.[chy]$)|(\\.tex$)|(Makefile)|(README)|(LICENSE)";
           package = tools.headache;
-          entry =
-            ## NOTE: `headache` made into in nixpkgs on 12 April 2023. At the
-            ## next NixOS release, the following code will become irrelevant.
-            lib.throwIf
-              (hooks.headache.package == null)
-              "The version of nixpkgs used by git-hooks.nix does not have `ocamlPackages.headache`. Please use a more recent version of nixpkgs."
-              "${hooks.headache.package}/bin/headache -h ${hooks.headache.settings.header-file}";
+          entry = "${hooks.headache.package}/bin/headache -h ${hooks.headache.settings.header-file}";
         };
       hindent =
         {
@@ -3646,16 +3638,7 @@ lib.escapeShellArgs (lib.concatMap (ext: [ "--ghc-opt" "-X${ext}" ]) hooks.fourm
       pre-commit-hook-ensure-sops = {
         name = "pre-commit-hook-ensure-sops";
         package = tools.pre-commit-hook-ensure-sops;
-        entry =
-          ## NOTE: pre-commit-hook-ensure-sops landed in nixpkgs on 8 July 2022. Once it reaches a
-          ## release of NixOS, the `throwIf` piece of code below will become
-          ## useless.
-          lib.throwIf
-            (hooks.pre-commit-hook-ensure-sops.package == null)
-            "The version of nixpkgs used by git-hooks.nix does not have the `pre-commit-hook-ensure-sops` package. Please use a more recent version of nixpkgs."
-            ''
-              ${hooks.pre-commit-hook-ensure-sops.package}/bin/pre-commit-hook-ensure-sops
-            '';
+        entry = "${hooks.pre-commit-hook-ensure-sops.package}/bin/pre-commit-hook-ensure-sops";
         files = "^secrets";
       };
       # See all CLI flags for prettier [here](https://prettier.io/docs/en/cli.html).
@@ -4134,25 +4117,17 @@ lib.escapeShellArgs (lib.concatMap (ext: [ "--ghc-opt" "-X${ext}" ]) hooks.fourm
           description = "A universal formatter engine within the Tree-sitter ecosystem, with support for many languages.";
           package = tools.topiary;
           entry =
-            ## NOTE: Topiary landed in nixpkgs on 2 Dec 2022. Once it reaches a
-            ## release of NixOS, the `throwIf` piece of code below will become
-            ## useless.
-            lib.throwIf
-              (hooks.topiary.package == null)
-              "The version of nixpkgs used by git-hooks.nix does not have the `topiary` package. Please use a more recent version of nixpkgs."
-              (
-                let
-                  topiary-inplace = pkgs.writeShellApplication {
-                    name = "topiary-inplace";
-                    text = ''
-                      for file; do
-                        ${hooks.topiary.package}/bin/topiary --in-place --input-file "$file"
-                      done
-                    '';
-                  };
-                in
-                "${topiary-inplace}/bin/topiary-inplace"
-              );
+            let
+              topiary-inplace = pkgs.writeShellApplication {
+                name = "topiary-inplace";
+                text = ''
+                  for file; do
+                    ${hooks.topiary.package}/bin/topiary --in-place --input-file "$file"
+                  done
+                '';
+              };
+            in
+            "${topiary-inplace}/bin/topiary-inplace";
           files = "(\\.json$)|(\\.toml$)|(\\.mli?$)";
         };
       treefmt =
@@ -4264,11 +4239,7 @@ lib.escapeShellArgs (lib.concatMap (ext: [ "--ghc-opt" "-X${ext}" ]) hooks.fourm
         name = "typstyle";
         description = "Beautiful and reliable typst code formatter";
         package = tools.typstyle;
-        entry =
-          lib.throwIf
-            (hooks.typstyle.package == null)
-            "The version of nixpkgs used by git-hooks.nix must contain typstyle"
-            "${hooks.typstyle.package}/bin/typstyle -i";
+        entry = "${hooks.typstyle.package}/bin/typstyle -i";
         files = "\\.typ$";
       };
       uv-check = {

nix/call-tools.nix

@@ -1,7 +1,17 @@
 pkgs:
-pkgs.lib.flip builtins.removeAttrs [ "override" "overrideDerivation" ]
-  (pkgs.callPackage ./tools.nix {
-    cabal-fmt = (pkgs.haskell.lib.enableSeparateBinOutput pkgs.haskellPackages.cabal-fmt).bin;
-    cabal-gild = (pkgs.haskell.lib.enableSeparateBinOutput pkgs.haskellPackages.cabal-gild).bin;
-    hindent = pkgs.haskell.lib.enableSeparateBinOutput pkgs.haskellPackages.hindent;
-  })
+pkgs.lib.flip builtins.removeAttrs [ "override" "overrideDerivation" ] (
+  pkgs.callPackage ./tools.nix {
+    placeholder = name: {
+      # Allows checking without forcing evaluation
+      meta.isPlaceholder = true;
+
+      # Throw when the package is actually used
+      outPath = throw ''
+        git-hooks: the package `${name} is not available in your nixpkgs revision.
+      '';
+
+      type = "derivation";
+      name = name + "placeholder";
+    };
+  }
+)

nix/default.nix

@@ -19,16 +19,46 @@ let
           ;
       };
 
-      # Filter out any broken or missing packages from our tests.
-      filterBrokenPackages = n: package: package != null && !(package.meta.broken or false);
+      # Filter out broken and placeholder packages.
+      filterPackageForCheck =
+        name: package:
+        let
+          isPlaceholder = package.meta.isPlaceholder or false;
+          isBroken = package.meta.broken or false;
+
+          check = builtins.tryEval (!(isPlaceholder || isBroken));
+          result = check.success && check.value;
+
+          message =
+            if !check.success then
+              ''
+                Skipping ${name} because it failed to evaluate.
+              ''
+            else if !check.value && isPlaceholder then
+              ''
+                Skipping ${name} because it is missing from this nixpkgs revision.
+              ''
+            else if !check.value && isBroken then
+              ''
+                Skipping ${name} because it is marked as broken.
+              ''
+            else
+              ""; # Not used
+
+        in
+        lib.warnIf (!result) message result;
     in
     {
       inherit tools run;
-      # Flake style attributes
-      packages = (lib.filterAttrs filterBrokenPackages tools) // {
+
+      # Flake-style attributes
+      # Do not remove: these are exposed in the flake.
+      # Each should also be a valid derivation.
+      packages = tools // {
         inherit (pkgs) pre-commit;
       };
-      checks = self.packages // {
+
+      checks = (lib.filterAttrs filterPackageForCheck tools) // {
         # A pre-commit-check for nix-pre-commit itself
         pre-commit-check = run {
           src = ../.;
@@ -57,7 +87,7 @@ let
               f n h.package;
 
             allEntryPoints = lib.pipe allHooks [
-              (lib.filterAttrs (getPackage filterBrokenPackages))
+              (lib.filterAttrs (getPackage filterPackageForCheck))
               (lib.mapAttrsToList getEntry)
             ];
           in