diff --git a/tests/default.nix b/tests/default.nix
index df0c4901..27faca17 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -43,16 +43,7 @@ lib.pipe (testFiles ++ [ exampleFiles ]) [
     let
       # The test case can either be the actual definition,
       # or a child attr named `module`.
-      prepareModule =
-        case: if lib.isFunction case then case else case.module or (lib.removeAttrs case [ "tests" ]);
-
-      # Convert legacy `dontRun` definitions into `test.runNvim` option defs
-      dontRunModule =
-        case:
-        let
-          dontRun = case.tests.dontRun or false;
-        in
-        lib.optionalAttrs dontRun { test.runNvim = false; };
+      prepareModule = case: if lib.isFunction case then case else case.module or case;
 
       mkTest =
         { name, case }:
@@ -60,12 +51,7 @@ lib.pipe (testFiles ++ [ exampleFiles ]) [
           inherit name;
           path = mkTestDerivationFromNixvimModule {
             inherit name;
-            module = {
-              imports = [
-                (dontRunModule case)
-                (prepareModule case)
-              ];
-            };
+            module = prepareModule case;
             pkgs = pkgsUnfree;
           };
         };
diff --git a/tests/test-sources/plugins/ai/chatgpt.nix b/tests/test-sources/plugins/ai/chatgpt.nix
index 08d5b443..8094fa02 100644
--- a/tests/test-sources/plugins/ai/chatgpt.nix
+++ b/tests/test-sources/plugins/ai/chatgpt.nix
@@ -1,12 +1,12 @@
 # We cannot run the test as loading the plugin requires a valid OPENAI_API_KEY
 {
   empty = {
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.chatgpt.enable = true;
   };
 
   defaults = {
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.chatgpt = {
       enable = true;
 
@@ -198,7 +198,7 @@
   };
 
   example = {
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.chatgpt = {
       enable = true;
 
diff --git a/tests/test-sources/plugins/completion/cmp-ai.nix b/tests/test-sources/plugins/completion/cmp-ai.nix
index 4c9056c0..389fb5e7 100644
--- a/tests/test-sources/plugins/completion/cmp-ai.nix
+++ b/tests/test-sources/plugins/completion/cmp-ai.nix
@@ -1,7 +1,7 @@
 {
   empty = {
     # We do not provide the required HF_API_KEY environment variable.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.cmp = {
       enable = true;
@@ -11,7 +11,7 @@
 
   example = {
     # We do not provide the required HF_API_KEY environment variable.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins = {
       cmp = {
diff --git a/tests/test-sources/plugins/completion/codeium-vim.nix b/tests/test-sources/plugins/completion/codeium-vim.nix
index 0015a1bf..b1ee1a00 100644
--- a/tests/test-sources/plugins/completion/codeium-vim.nix
+++ b/tests/test-sources/plugins/completion/codeium-vim.nix
@@ -1,14 +1,14 @@
 {
   empty = {
     # For some reason, nvim hangs when using codeium-vim. After checking, it doesn't look like a bug though.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.codeium-vim.enable = true;
   };
 
   example = {
     # For some reason, nvim hangs when using codeium-vim. After checking, it doesn't look like a bug though.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.codeium-vim = {
       enable = true;
diff --git a/tests/test-sources/plugins/git/octo.nix b/tests/test-sources/plugins/git/octo.nix
index 5809c8db..f3846e42 100644
--- a/tests/test-sources/plugins/git/octo.nix
+++ b/tests/test-sources/plugins/git/octo.nix
@@ -1,14 +1,14 @@
 {
   empty = {
     # This test is flaky and fails non-deterministically
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.octo.enable = true;
   };
 
   example = {
     # This test is flaky and fails non-deterministically
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.octo = {
       enable = true;
@@ -28,7 +28,7 @@
 
   withFzfLuaPicker = {
     # This test is flaky and fails non-deterministically
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.octo = {
       enable = true;
@@ -38,7 +38,7 @@
 
   defaults = {
     # This test is flaky and fails non-deterministically
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.octo = {
       enable = true;
diff --git a/tests/test-sources/plugins/languages/treesitter/treesitter.nix b/tests/test-sources/plugins/languages/treesitter/treesitter.nix
index d3776d7a..37c2edd6 100644
--- a/tests/test-sources/plugins/languages/treesitter/treesitter.nix
+++ b/tests/test-sources/plugins/languages/treesitter/treesitter.nix
@@ -83,7 +83,7 @@
 
   no-nix = {
     # TODO: See if we can build parsers (legacy way)
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.treesitter = {
       enable = true;
       nixGrammars = false;
diff --git a/tests/test-sources/plugins/neotest/gtest.nix b/tests/test-sources/plugins/neotest/gtest.nix
index 23e4effa..f6bcd602 100644
--- a/tests/test-sources/plugins/neotest/gtest.nix
+++ b/tests/test-sources/plugins/neotest/gtest.nix
@@ -2,7 +2,7 @@
   example = {
     # We cannot test neotest-gtest as it tries to create file in the upper directory
     # https://github.com/alfaix/neotest-gtest/blob/6e794ac91f4c347e2ea5ddeb23d594f8fc64f2a8/lua/neotest-gtest/utils.lua#L10-L16
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins = {
       treesitter.enable = true;
diff --git a/tests/test-sources/plugins/telescope/frecency.nix b/tests/test-sources/plugins/telescope/frecency.nix
index d107df1e..f0e3f361 100644
--- a/tests/test-sources/plugins/telescope/frecency.nix
+++ b/tests/test-sources/plugins/telescope/frecency.nix
@@ -1,7 +1,7 @@
 {
   empty = {
     # A warning is displayed on stdout
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.telescope = {
       enable = true;
@@ -11,7 +11,7 @@
 
   defaults = {
     # A warning is displayed on stdout
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.telescope = {
       enable = true;
@@ -46,7 +46,7 @@
 
   example = {
     # A warning is displayed on stdout
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.telescope = {
       enable = true;
diff --git a/tests/test-sources/plugins/ui/image.nix b/tests/test-sources/plugins/ui/image.nix
index a0909938..8cfa5b3f 100644
--- a/tests/test-sources/plugins/ui/image.nix
+++ b/tests/test-sources/plugins/ui/image.nix
@@ -2,7 +2,7 @@
   empty = {
     # At runtime, the plugin tries to get the size of the terminal which doesn't exist in the
     # headless environment.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.image.enable = true;
   };
@@ -10,7 +10,7 @@
   defaults = {
     # At runtime, the plugin tries to get the size of the terminal which doesn't exist in the
     # headless environment.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.image = {
       enable = true;
@@ -67,7 +67,7 @@
   ueberzug-backend = {
     # At runtime, the plugin tries to get the size of the terminal which doesn't exist in the
     # headless environment.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.image = {
       enable = true;
diff --git a/tests/test-sources/plugins/utils/harpoon.nix b/tests/test-sources/plugins/utils/harpoon.nix
index cd9a7007..d691c5a2 100644
--- a/tests/test-sources/plugins/utils/harpoon.nix
+++ b/tests/test-sources/plugins/utils/harpoon.nix
@@ -2,7 +2,7 @@
   empty = {
     # Harpoon expects to access `~/.local/share/nvim/harpoon.json` which is not available in the
     # test environment
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.harpoon.enable = true;
   };
@@ -10,7 +10,7 @@
   telescopeEnabled = {
     # Harpoon expects to access `~/.local/share/nvim/harpoon.json` which is not available in the
     # test environment
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.telescope = {
       enable = true;
@@ -75,7 +75,7 @@
   telescopeDisabled = {
     # Harpoon expects to access `~/.local/share/nvim/harpoon.json` which is not available in the
     # test environment
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.harpoon = {
       enable = true;
diff --git a/tests/test-sources/plugins/utils/neocord.nix b/tests/test-sources/plugins/utils/neocord.nix
index caaf7fab..00c9c2e0 100644
--- a/tests/test-sources/plugins/utils/neocord.nix
+++ b/tests/test-sources/plugins/utils/neocord.nix
@@ -1,13 +1,13 @@
 {
   empty = {
     # don't run tests as they try to access the network.
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.neocord.enable = true;
   };
 
   defaults = {
     # don't run tests as they try to access the network.
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.neocord = {
       enable = true;
 
@@ -39,7 +39,7 @@
 
   example = {
     # don't run tests as they try to access the network.
-    tests.dontRun = true;
+    test.runNvim = false;
     plugins.neocord = {
       enable = true;
 
diff --git a/tests/test-sources/plugins/utils/neorg.nix b/tests/test-sources/plugins/utils/neorg.nix
index 26901ba5..9f6195ea 100644
--- a/tests/test-sources/plugins/utils/neorg.nix
+++ b/tests/test-sources/plugins/utils/neorg.nix
@@ -3,7 +3,7 @@
     # neorg should be re-packaged in nixpkgs from the luarocks `neorg` package.
     # In the meantime, disable the test.
     # TODO: re-enable when this will have been fixed upstream.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.neorg.enable = true;
   };
@@ -12,7 +12,7 @@
     # neorg should be re-packaged in nixpkgs from the luarocks `neorg` package.
     # In the meantime, disable the test.
     # TODO: re-enable when this will have been fixed upstream.
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins = {
       # Treesitter is required when using the "core.defaults" module.
diff --git a/tests/test-sources/plugins/utils/rest.nix b/tests/test-sources/plugins/utils/rest.nix
index 57465aa1..3cc3b1c2 100644
--- a/tests/test-sources/plugins/utils/rest.nix
+++ b/tests/test-sources/plugins/utils/rest.nix
@@ -3,7 +3,7 @@
     # As of 2024-05-07, the lua dependencies of luaPackage.rest-nvim are not correctly propagated to
     # the vim plugin.
     # TODO: re-enable this test when this issue will have been fixed
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.rest.enable = true;
   };
@@ -12,7 +12,7 @@
     # As of 2024-05-07, the lua dependencies of luaPackage.rest-nvim are not correctly propagated to
     # the vim plugin.
     # TODO: re-enable this test when this issue will have been fixed
-    tests.dontRun = true;
+    test.runNvim = false;
 
     plugins.rest = {
       enable = true;