modules/test: allow arbitrary warning/assertion expecations

Allow defining expectation predicates that will be tested on the final
warnings/assertions lists.

modules/top-level/test.nix

@@ -8,8 +8,60 @@
 let
   cfg = config.test;
 
-  inherit (config) warnings;
-  assertions = builtins.concatMap (x: lib.optional (!x.assertion) x.message) config.assertions;
+  # Expectation submodule used for checking warnings and/or assertions
+  expectationType = lib.types.submodule (
+    { config, ... }:
+    let
+      # NOTE: ensure `config.expect != null` before evaluating this!
+      namedPredicate = cfg.namedExpectationPredicates.${config.expect};
+    in
+    {
+      options = {
+        predicate = lib.mkOption {
+          type = with lib.types; functionTo bool;
+          description = ''
+            A predicate of that determines whether this expectation is met.
+
+            Type
+            ```
+            [String] -> Boolean
+            ```
+
+            Parameters
+            1. values - all warnings or matched assertions
+          '';
+        };
+        expect = lib.mkOption {
+          type = with lib.types; nullOr (enum (builtins.attrNames cfg.namedExpectationPredicates));
+          description = ''
+            If non-null, will be used together with `value` to define `predicate`.
+          '';
+          default = null;
+        };
+        value = lib.mkOption {
+          type = lib.types.anything;
+          description = ''
+            If defined, will be used together with `expect` to define `predicate`.
+          '';
+        };
+        message = lib.mkOption {
+          type = with lib.types; either str (functionTo str);
+          description = ''
+            The assertion message.
+
+            If the value is a function, it is called with the same list of warning/assertion messages that is applied to `predicate`.
+          '';
+          defaultText = lib.literalMD ''
+            If `assertion` is non-null, a default is computed. Otherwise, there is no default.
+          '';
+        };
+      };
+      config = lib.mkIf (config.expect != null) {
+        predicate = lib.mkOptionDefault (namedPredicate.predicate config.value);
+        message = lib.mkOptionDefault (namedPredicate.message config.value);
+      };
+    }
+  );
 in
 {
   options.test = {
@@ -43,6 +95,55 @@ in
       description = "Whether to check `config.assertions` in the test.";
       default = true;
     };
+
+    warnings = lib.mkOption {
+      type = lib.types.listOf expectationType;
+      description = "A list of expectations for `warnings`.";
+      defaultText = lib.literalMD "Expect `count == 0`";
+      default = [
+        {
+          expect = "count";
+          value = 0;
+        }
+      ];
+    };
+
+    assertions = lib.mkOption {
+      type = lib.types.listOf expectationType;
+      description = "A list of expectations for `assertions`.";
+      defaultText = lib.literalMD "Expect `count == 0`";
+      default = [
+        {
+          expect = "count";
+          value = 0;
+        }
+      ];
+    };
+
+    namedExpectationPredicates = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (submodule {
+          options = {
+            predicate = lib.mkOption {
+              type = functionTo (functionTo bool);
+              description = ''
+                Predicate matching `(value) -> [(message)] -> Boolean`.
+              '';
+            };
+            message = lib.mkOption {
+              type = functionTo (either str (functionTo str));
+              description = ''
+                Expectation message supplier, matching `(value) -> String` or `(value) -> [(message)] -> String`.
+              '';
+            };
+          };
+        });
+      description = ''
+        A list of named expectation predicates, for use with `test.warnings.*.expect` and `test.assertions.*.expect`.
+      '';
+      internal = true;
+    };
   };
 
   options.build = {
@@ -57,23 +158,56 @@ in
 
   config =
     let
-      showErr =
-        name: lines:
-        lib.optionalString (lines != [ ]) ''
-          Unexpected ${name}:
-          ${lib.concatStringsSep "\n" (lib.map (v: "- ${v}") lines)}
-        '';
+      input = {
+        inherit (config) warnings;
+        assertions = builtins.concatMap (x: lib.optional (!x.assertion) x.message) config.assertions;
+      };
 
-      toCheck =
-        lib.optionalAttrs cfg.checkWarnings { inherit warnings; }
-        // lib.optionalAttrs cfg.checkAssertions { inherit assertions; };
+      expectationMessages =
+        name:
+        lib.pipe cfg.${name} [
+          (builtins.filter (x: !x.predicate input.${name}))
+          (builtins.map (x: x.message))
+          (builtins.map (msg: if lib.isFunction msg then msg input.${name} else msg))
+          (
+            x:
+            if x == [ ] then
+              null
+            else
+              ''
+                Failed ${toString (builtins.length x)} expectation${lib.optionalString (builtins.length x > 1) "s"}:
+                ${lib.concatMapStringsSep "\n" (line: "- ${line}") x}
+                For ${name}:
+                ${lib.concatMapStringsSep "\n" (line: "- ${line}") input.${name}}
+              ''
+          )
+        ];
 
-      errors = lib.foldlAttrs (
-        err: name: lines:
-        err + showErr name lines
-      ) "" toCheck;
+      failedExpectations = lib.genAttrs [ "warnings" "assertions" ] expectationMessages;
     in
     {
+      test = {
+        # If checkWarnings or checkAssertions are disabled, ensure the default expectations are overridden
+        assertions = lib.mkIf (!cfg.checkAssertions) [ ];
+        warnings = lib.mkIf (!cfg.checkWarnings) [ ];
+
+        # Expectation predicates available via the `expect` enum-option
+        namedExpectationPredicates = {
+          count = {
+            predicate = v: l: builtins.length l == v;
+            message = v: l: "Expected length to be ${toString v} but found ${toString (builtins.length l)}.";
+          };
+          any = {
+            predicate = v: builtins.any (lib.hasInfix v);
+            message = v: "Expected ${lib.toJSON v} infix to be present.";
+          };
+          anyExact = {
+            predicate = builtins.elem;
+            message = v: "Expected ${lib.toJSON v} to be present.";
+          };
+        };
+      };
+
       build.test =
         assert lib.assertMsg (cfg.runNvim -> cfg.buildNixvim) "`test.runNvim` requires `test.buildNixvim`.";
         pkgs.runCommandNoCCLocal cfg.name
@@ -82,6 +216,8 @@ in
               config.build.packageUnchecked
             ];
 
+            inherit (failedExpectations) warnings assertions;
+
             # Allow inspecting the test's module a little from the repl
             # e.g.
             # :lf .
@@ -94,9 +230,15 @@ in
           }
           (
             # First check warnings/assertions, then run nvim
-            lib.optionalString (errors != "") ''
-              echo -n ${lib.escapeShellArg errors}
-              exit 1
+            ''
+              if [ -n "$warnings" ]; then
+                echo -n "$warnings"
+                exit 1
+              fi
+              if [ -n "$assertions" ]; then
+                echo -n "$assertions"
+                exit 1
+              fi
             ''
             # We need to set HOME because neovim will try to create some files
             #

tests/failing-tests.nix

@@ -38,7 +38,9 @@ linkFarmFromDrvs "failing-tests" [
     }
     ''
       [[ 1 = $(cat "$failed/testBuildFailure.exit") ]]
-      grep -F 'Unexpected warnings:' "$failed/testBuildFailure.log"
+      grep -F 'Failed 1 expectation' "$failed/testBuildFailure.log"
+      grep -F 'Expected length to be 0 but found 1.' "$failed/testBuildFailure.log"
+      grep -F 'For warnings' "$failed/testBuildFailure.log"
       grep -F 'Hello, world!' "$failed/testBuildFailure.log"
       touch $out
     ''
@@ -59,7 +61,9 @@ linkFarmFromDrvs "failing-tests" [
     }
     ''
       [[ 1 = $(cat "$failed/testBuildFailure.exit") ]]
-      grep -F 'Unexpected assertions:' "$failed/testBuildFailure.log"
+      grep -F 'Failed 1 expectation' "$failed/testBuildFailure.log"
+      grep -F 'Expected length to be 0 but found 1.' "$failed/testBuildFailure.log"
+      grep -F 'For assertions' "$failed/testBuildFailure.log"
       grep -F 'Hello, world!' "$failed/testBuildFailure.log"
       touch $out
     ''