diff --git a/flake.nix b/flake.nix
index 92852b2a..df441ca5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -62,6 +62,10 @@
             };
           extractRustAnalyzerPkg = pkgs.callPackage extractRustAnalyzer {};
         in {
+          checks = import ./tests/checks.nix {
+            inherit pkgs;
+            makeNixvim = self.legacyPackages."${system}".makeNixvim;
+          };
           packages = {
             docs = pkgs.callPackage (import ./docs.nix) {
               modules = nixvimModules;
diff --git a/tests/checks.nix b/tests/checks.nix
new file mode 100644
index 00000000..3c127d7a
--- /dev/null
+++ b/tests/checks.nix
@@ -0,0 +1,36 @@
+{
+  makeNixvim,
+  pkgs,
+}: let
+  tests = import ./plugins {inherit (pkgs) lib;};
+in
+  # We attempt to build & execute all configurations
+  builtins.mapAttrs (
+    name: config: let
+      nvim = makeNixvim config;
+    in
+      pkgs.stdenv.mkDerivation {
+        name = name;
+
+        nativeBuildInputs = [nvim];
+
+        dontUnpack = true;
+        # We need to set HOME because neovim will try to create some files
+        #
+        # Because neovim does not return an exitcode when quitting we need to check if there are
+        # errors on stderr
+        buildPhase = ''
+          output=$(HOME=$(realpath .) nvim -mn --headless "+q" 2>&1 >/dev/null)
+          if [[ -n $output ]]; then
+          	echo "ERROR: $output"
+            exit 1
+          fi
+        '';
+
+        # If we don't do this nix is not happy
+        installPhase = ''
+          mkdir $out
+        '';
+      }
+  )
+  tests
diff --git a/tests/plugins/default.nix b/tests/plugins/default.nix
new file mode 100644
index 00000000..d7483444
--- /dev/null
+++ b/tests/plugins/default.nix
@@ -0,0 +1,23 @@
+{lib}: let
+  # List of files containing configurations
+  pluginFiles =
+    builtins.filter (p: p != "default.nix") (builtins.attrNames (builtins.readDir ./.));
+
+  /*
+  Create a list of tests. The list is of the form:
+    [ { name = "<plugin>-<test_name>"; value = { ... }; } ]
+  */
+  makePluginTests = pluginFile: let
+    pluginName = builtins.head (lib.strings.splitString "." pluginFile);
+    pluginConfigs = import (./. + "/${pluginFile}");
+  in
+    lib.attrsets.mapAttrsToList (testName: testConfig: {
+      name = "${pluginName}-${testName}";
+      value = testConfig;
+    })
+    pluginConfigs;
+
+  # A list of lists of test cases for each plugin
+  pluginTests = builtins.map makePluginTests pluginFiles;
+in
+  builtins.listToAttrs (lib.lists.flatten pluginTests)
diff --git a/tests/plugins/tokyonight.nix b/tests/plugins/tokyonight.nix
new file mode 100644
index 00000000..0c01364e
--- /dev/null
+++ b/tests/plugins/tokyonight.nix
@@ -0,0 +1,31 @@
+{
+  # All the upstream default options of tokyonight
+  defaults = {
+    colorschemes.tokyonight = {
+      enable = true;
+
+      style = "storm";
+      # Not implemented
+      # lightStyle = "day";
+      transparent = false;
+      terminalColors = true;
+      styles = {
+        comments = {italic = true;};
+        keywords = {italic = true;};
+        functions = {};
+        variables = {};
+        sidebars = "dark";
+        floats = "dark";
+      };
+      sidebars = ["qf" "help"];
+      dayBrightness = 0.3;
+      hideInactiveStatusline = false;
+      dimInactive = false;
+      lualineBold = false;
+      # Not implemented
+      # onColors = {__raw = "function(colors) end";};
+      # Not implemented
+      # onHighlights = {__raw = "function(colors) end";};
+    };
+  };
+}