tests/generated: init by checking declared tools

This moves most assertions out of generate-files and into a check
derivation. This should allow the CI to finish, even when there are
issues.

This also properly tests efmls, which was only checked partially before.

rust-analyzer is not covered because the existing assertions relate more
to edge-cases not handled by the generation script than the result it
builds.

flake-modules/tests.nix

@@ -38,6 +38,8 @@
           };
 
           maintainers = import ../tests/maintainers.nix { inherit pkgs; };
+
+          generated = pkgs.callPackage ../tests/generated.nix { };
         }
         // (import ../tests {
           inherit

flake-modules/updates/efmls-configs.nix

@@ -2,7 +2,6 @@
   lib,
   vimPlugins,
   writeText,
-  pkgs,
 }:
 let
   tools = lib.trivial.importJSON "${vimPlugins.efmls-configs-nvim.src}/doc/supported-list.json";
@@ -47,22 +46,8 @@ let
         };
       };
     };
-
-  inherit (import ../../plugins/lsp/language-servers/efmls-configs-pkgs.nix pkgs) packaged unpackaged;
-
-  toolList = lib.lists.unique (
-    lib.concatLists (
-      lib.map ({ linter, formatter }: linter.possible ++ formatter.possible) (lib.attrValues sources)
-    )
-  );
-
-  unknownTools = lib.filter (tool: !(lib.hasAttr tool packaged || lib.elem tool unpackaged)) toolList;
 in
 writeText "efmls-configs-sources.nix" (
-  assert lib.assertMsg (lib.length unknownTools == 0)
-    "The following tools are neither marked as unpackaged nor as packaged: ${
-      lib.generators.toPretty { } unknownTools
-    }";
   "# WARNING: DO NOT EDIT\n"
   + "# This file is generated with packages.<system>.efmls-configs-sources, which is run automatically by CI\n"
   + (lib.generators.toPretty { } sources)

flake-modules/updates/none-ls.nix

@@ -2,30 +2,12 @@
   vimPlugins,
   lib,
   writeText,
-  pkgs,
 }:
 let
-  inherit (import ../../plugins/none-ls/packages.nix pkgs) packaged noPackage;
-
   builtinSources = lib.trivial.importJSON "${vimPlugins.none-ls-nvim.src}/doc/builtins.json";
-
   builtinSourceNames = lib.mapAttrs (_: lib.attrNames) builtinSources;
-
-  toolNames = lib.unique (lib.flatten (lib.attrValues builtinSourceNames));
-
-  undeclaredTool = lib.filter (
-    name: !(lib.hasAttr name packaged || lib.elem name noPackage)
-  ) toolNames;
-
-  uselesslyDeclaredTool = lib.filter (name: !(lib.elem name toolNames)) (
-    noPackage ++ (lib.attrNames packaged)
-  );
 in
-writeText "efmls-configs-sources.nix" (
-  assert lib.assertMsg (lib.length undeclaredTool == 0)
-    "Undeclared tools: ${lib.generators.toPretty { } undeclaredTool}";
-  assert lib.assertMsg (lib.length uselesslyDeclaredTool == 0)
-    "Tool is not supported upstream: ${lib.generators.toPretty { } uselesslyDeclaredTool}";
+writeText "none-ls-sources.nix" (
   "# WARNING: DO NOT EDIT\n"
   + "# This file is generated with packages.<system>.none-ls-builtins, which is run automatically by CI\n"
   + (lib.generators.toPretty { } builtinSourceNames)

tests/generated.nix

@@ -0,0 +1,77 @@
+{
+  lib,
+  runCommand,
+  pkgs,
+}:
+let
+  # Format a list of errors with an error message and trailing newline
+  describeErrors =
+    desc: errors:
+    lib.optionals (errors != [ ]) (lib.toList desc ++ lib.map (v: "- ${v}") errors ++ [ "" ]);
+
+  # Build error messages for the given declared & generated names
+  checkDeclarations =
+    {
+      # The plugin's name
+      name,
+      # A list of names declared in declarationFile
+      declared,
+      # A list of names generated by generate-files
+      generated,
+      # The filename where names are declared (used in error messages)
+      declarationFile,
+    }:
+    let
+      undeclared = lib.filter (name: !(lib.elem name declared)) generated;
+      uselesslyDeclared = lib.filter (name: !(lib.elem name generated)) declared;
+    in
+    describeErrors "${name}: The following are not declared in ${declarationFile}:" undeclared
+    ++ describeErrors "${name}: The following are not listed upstream, but are declared in ${declarationFile}:" uselesslyDeclared;
+
+  # The error message provided to the derivation.
+  # The test fails if this is non-empty.
+  errors = lib.concatStringsSep "\n" (
+    checkDeclarations {
+      name = "none-ls";
+      declarationFile = "plugins/none-ls/packages.nix";
+
+      declared =
+        let
+          inherit (import ../plugins/none-ls/packages.nix pkgs) noPackage packaged;
+        in
+        noPackage ++ lib.attrsets.attrNames packaged;
+
+      generated = lib.pipe ../generated/none-ls.nix [
+        import
+        lib.attrsets.attrValues
+        lib.lists.concatLists
+        lib.lists.unique
+      ];
+    }
+    ++ checkDeclarations {
+      name = "efmls";
+      declarationFile = "efmls-configs-pkgs.nix";
+
+      declared =
+        let
+          inherit (import ../plugins/lsp/language-servers/efmls-configs-pkgs.nix pkgs) packaged unpackaged;
+        in
+        unpackaged ++ lib.attrsets.attrNames packaged;
+
+      generated = lib.pipe ../generated/efmls-configs.nix [
+        import
+        lib.attrsets.attrValues
+        (lib.map ({ linter, formatter }: linter.possible ++ formatter.possible))
+        lib.lists.concatLists
+        lib.lists.unique
+      ];
+    }
+  );
+in
+runCommand "generated-sources-test" { inherit errors; } ''
+  if [ -n "$errors" ]; then
+    echo -n "$errors"
+    exit 1
+  fi
+  touch "$out"
+''