modules/performance: add ability to byte compile lua configuration files

This commit adds support for byte compiling lua configuration files.
It's enabled by default (if byte compiling is enabled at all) and can be
disabled with `performance.byteCompileLua.configs` toggle.

To implement this feature `extraFiles.<name>.finalSource` internal
read-only option is introduced. `source` option cannot be used because
it's user configurable. In order to access the values of the
`performance.byteCompileLua` options, parent config is added to
specialArgs of extraFiles submodule. Then the usages of `source` option
changed to `finalSource` in all relevant places (filesPlugin and
wrappers).

Added more helpers for various cases of byte compiling:

* `byteCompileLuaFile` byte compiles lua file
* `byteCompileLuaHook` is a setup hook that byte compiles all lua files
* `byteCompileLuaDrv` overrides derivation by adding byteCompileLuaHook
  to it

Added tests to validate that extraFiles specified by various methods are
handled correctly. Added a separate home-manager test, that is intended
to validate that extraFiles propagated to wrapper modules are correctly
byte compiled.

flake-modules/wrappers.nix

@@ -15,6 +15,13 @@
               inherit (inputs) home-manager;
               nixvim = self;
             }).activationPackage;
+          home-manager-extra-files-byte-compiling =
+            import ../tests/modules/hm-extra-files-byte-compiling.nix
+              {
+                inherit pkgs;
+                inherit (inputs) home-manager;
+                nixvim = self;
+              };
         }
         // pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) {
           nixos-module =

lib/builders.nix

@@ -1,5 +1,5 @@
 { lib, pkgs }:
-{
+rec {
   /*
     Write a lua file to the nix store, formatted using stylua.
 
@@ -48,4 +48,88 @@
 
       ${lib.getExe' pkgs.luajit "luajit"} -bd -- "$out" "$out"
     '';
+
+  /*
+    Get a source lua file and write a byte compiled copy of it
+    to the nix store.
+
+    # Type
+
+    ```
+    byteCompileLuaFile :: String -> String -> Derivation
+    ```
+
+    # Arguments
+
+    - [name] The name of the derivation
+    - [src] The path to the source lua file
+  */
+  byteCompileLuaFile =
+    name: src:
+    pkgs.runCommandLocal name { inherit src; } ''
+      ${lib.getExe' pkgs.luajit "luajit"} -bd -- "$src" "$out"
+    '';
+
+  # Setup hook to byte compile all lua files in the output directory.
+  # Invalid lua files are ignored.
+  byteCompileLuaHook = pkgs.makeSetupHook { name = "byte-compile-lua-hook"; } (
+    let
+      luajit = lib.getExe' pkgs.luajit "luajit";
+    in
+    pkgs.writeText "byte-compile-lua-hook.sh" # bash
+      ''
+        byteCompileLuaPostFixup() {
+            # Target is a single file
+            if [[ -f $out ]]; then
+                if [[ $out = *.lua ]]; then
+                    tmp=$(mktemp)
+                    ${luajit} -bd -- "$out" "$tmp"
+                    mv "$tmp" "$out"
+                fi
+                return
+            fi
+
+            # Target is a directory
+            while IFS= read -r -d "" file; do
+                tmp=$(mktemp -u "$file.XXXX")
+                # Ignore invalid lua files
+                if ${luajit} -bd -- "$file" "$tmp"; then
+                    mv "$tmp" "$file"
+                else
+                    echo "WARNING: Ignoring byte compiling error for '$file' lua file" >&2
+                fi
+            done < <(find "$out" -type f,l -name "*.lua" -print0)
+        }
+
+        postFixupHooks+=(byteCompileLuaPostFixup)
+      ''
+  );
+
+  /*
+    Returns an overridden derivation with all lua files byte compiled.
+
+    # Type
+
+    ```
+    byteCompileLuaDrv :: Derivation -> Derivation
+    ```
+
+    # Arguments
+
+    - [drv] Input derivation
+  */
+  byteCompileLuaDrv =
+    drv:
+    drv.overrideAttrs (
+      prev:
+      {
+        nativeBuildInputs = prev.nativeBuildInputs or [ ] ++ [ byteCompileLuaHook ];
+      }
+      // lib.optionalAttrs (prev ? buildCommand) {
+        buildCommand = ''
+          ${prev.buildCommand}
+          runHook postFixup
+        '';
+      }
+    );
 }

modules/files.nix

@@ -2,14 +2,16 @@
   lib,
   helpers,
   pkgs,
+  config,
   ...
 }:
 let
-  fileType = lib.types.submodule (
+  fileTypeModule =
     {
       name,
       config,
       options,
+      topConfig,
       ...
     }:
     {
@@ -41,6 +43,14 @@ let
           type = lib.types.path;
           description = "Path of the source file.";
         };
+
+        finalSource = lib.mkOption {
+          type = lib.types.path;
+          description = "Path to the final source file.";
+          readOnly = true;
+          visible = false;
+          internal = true;
+        };
       };
 
       config =
@@ -54,9 +64,29 @@ let
             # This means our `source` definition has the same priority as `text`.
             lib.mkDerivedConfig options.text (pkgs.writeText derivationName)
           );
+          finalSource =
+            # Byte compile lua files if performance.byteCompileLua option is enabled
+            if
+              lib.hasSuffix ".lua" config.target
+              && topConfig.performance.byteCompileLua.enable
+              && topConfig.performance.byteCompileLua.configs
+            then
+              if lib.isDerivation config.source then
+                # Source is a derivation
+                helpers.byteCompileLuaDrv config.source
+              else
+                # Source is a path or string
+                helpers.byteCompileLuaFile derivationName config.source
+            else
+              config.source;
         };
-    }
-  );
+    };
+
+  fileType = lib.types.submoduleWith {
+    shorthandOnlyDefinesConfig = true;
+    modules = [ fileTypeModule ];
+    specialArgs.topConfig = config;
+  };
 
   # TODO: Added 2024-07-07, remove after 24.11
   # Before we had a fileType, we used types.str.

modules/performance.nix

@@ -12,6 +12,12 @@ in
         default = true;
         example = false;
       };
+      configs = lib.mkOption {
+        description = "Whether to byte compile lua configuration files.";
+        type = types.bool;
+        default = true;
+        example = false;
+      };
     };
 
     combinePlugins = {

modules/top-level/files/default.nix

@@ -84,11 +84,11 @@ in
 
         mkdir -p "$out"
         ${lib.concatMapStringsSep "\n" (
-          { target, source, ... }:
+          { target, finalSource, ... }:
           lib.escapeShellArgs [
             "makeEntry"
             # Force local source paths to be added to the store
-            "${source}"
+            "${finalSource}"
             target
           ]
         ) extraFiles}

tests/fetch-tests.nix

@@ -14,7 +14,7 @@ let
       file = "${root}/${relativePath}/${name}";
     in
     if type == "regular" then
-      [
+      lib.optional (lib.hasSuffix ".nix" name) [
         {
           namespace = namespace ++ [ (lib.strings.removeSuffix ".nix" name) ];
           cases = import file;

tests/modules/hm-extra-files-byte-compiling.nix

@@ -0,0 +1,104 @@
+{
+  nixvim,
+  pkgs,
+  home-manager,
+}:
+let
+  config = {
+    home = {
+      username = "nixvim";
+      homeDirectory = "/invalid/dir";
+      stateVersion = "24.05";
+    };
+
+    programs.nixvim = {
+      enable = true;
+
+      performance.byteCompileLua.enable = true;
+
+      extraFiles = {
+        "extra-files/test1.lua".text = "vim.opt.tabstop = 2";
+        "extra-files/test2.lua".source = builtins.toFile "file_source.lua" "vim.opt.tabstop = 2";
+        "extra-files/test3.lua".source = pkgs.writeText "test3.lua" "vim.opt.tabstop = 2";
+        "extra-files/test.vim".text = "set tabstop=2";
+        "extra-files/test.json".text = builtins.toJSON { a = 1; };
+      };
+
+      files = {
+        "files/test.lua".opts.tabstop = 2;
+        "files/test.vim".opts.tabstop = 2;
+      };
+    };
+  };
+
+  homeFilesByteCompilingEnabled =
+    (home-manager.lib.homeManagerConfiguration {
+      inherit pkgs;
+
+      modules = [
+        nixvim.homeManagerModules.nixvim
+        config
+        { programs.nixvim.performance.byteCompileLua.configs = true; }
+      ];
+    }).config.home-files;
+
+  homeFilesByteCompilingDisabled =
+    (home-manager.lib.homeManagerConfiguration {
+      inherit pkgs;
+
+      modules = [
+        nixvim.homeManagerModules.nixvim
+        config
+        { programs.nixvim.performance.byteCompileLua.configs = false; }
+      ];
+    }).config.home-files;
+in
+pkgs.runCommand "home-manager-extra-files-byte-compiling" { } ''
+  is_binary() {
+      ! grep -qI . "$1"
+  }
+  test_byte_compiled() {
+      if ! is_binary "$home_files/.config/nvim/$1"; then
+          echo "File $1 is expected to be byte compiled, but it's not"
+          exit 1
+      fi
+  }
+  test_not_byte_compiled() {
+      if is_binary "$home_files/.config/nvim/$1"; then
+          echo "File $1 is not expected to be byte compiled, but it is"
+          exit 1
+      fi
+  }
+
+  # Test directory with extraFiles byte compiling enabled
+  home_files="${homeFilesByteCompilingEnabled}"
+
+  echo "Testing home-files with extraFiles byte compiling enabled"
+
+  # extraFiles
+  test_byte_compiled extra-files/test1.lua
+  test_byte_compiled extra-files/test2.lua
+  test_byte_compiled extra-files/test3.lua
+  test_not_byte_compiled extra-files/test.vim
+  test_not_byte_compiled extra-files/test.json
+  # files
+  test_byte_compiled files/test.lua
+  test_not_byte_compiled files/test.vim
+
+  # Test directory with extraFiles byte compiling disabled
+  home_files="${homeFilesByteCompilingDisabled}"
+
+  echo "Testing home-files with extraFiles byte compiling disabled"
+
+  # extraFiles
+  test_not_byte_compiled extra-files/test1.lua
+  test_not_byte_compiled extra-files/test2.lua
+  test_not_byte_compiled extra-files/test3.lua
+  test_not_byte_compiled extra-files/test.vim
+  test_not_byte_compiled extra-files/test.json
+  # files
+  test_not_byte_compiled files/test.lua
+  test_not_byte_compiled files/test.vim
+
+  touch $out
+''

tests/test-sources/modules/performance/byte-compile-lua.nix

@@ -30,10 +30,31 @@ in
       performance.byteCompileLua.enable = true;
 
       extraFiles = {
+        # By text
         "plugin/file_text.lua".text = "vim.opt.tabstop = 2";
+        # By simple source derivation using buildCommand
         "plugin/file_source.lua".source = helpers.writeLua "file_source.lua" "vim.opt.tabstop = 2";
+        # By standard derivation, it needs to execute fixupPhase
+        "plugin/file_drv.lua".source = pkgs.stdenvNoCC.mkDerivation {
+          name = "file_drv.lua";
+          src = pkgs.emptyDirectory;
+          buildPhase = ''
+            echo "vim.opt.tabstop = 2" > $out
+          '';
+        };
+        # By path
+        "plugin/file_path.lua".source = ./files/file.lua;
+        # By string
+        "plugin/file_string.lua".source = builtins.toFile "file_path.lua" "vim.opt.tabstop = 2";
+        # By derivation converted to string
+        "plugin/file_drv_string.lua".source = toString (
+          helpers.writeLua "file_drv_string.lua" "vim.opt.tabstop = 2"
+        );
+        # Non-lua files
         "plugin/test.vim".text = "set tabstop=2";
         "plugin/test.json".text = builtins.toJSON { a = 1; };
+        # Lua file with txt extension won't be byte compiled
+        "test.txt".source = helpers.writeLua "test.txt" "vim.opt.tabstop = 2";
       };
 
       files = {
@@ -64,14 +85,19 @@ in
         local init_content = vim.fn.system("${config.printInitPackage}/bin/nixvim-print-init")
         assert(init_content:find("VALIDATING_STRING"), "nixvim-print-init's output is byte compiled")
 
-        -- extraFiles
-        test_rtp_file("plugin/file_text.lua", false)
-        test_rtp_file("plugin/file_source.lua", false)
+        -- lua extraFiles are byte compiled
+        test_rtp_file("plugin/file_text.lua", true)
+        test_rtp_file("plugin/file_source.lua", true)
+        test_rtp_file("plugin/file_drv.lua", true)
+        test_rtp_file("plugin/file_path.lua", true)
+        test_rtp_file("plugin/file_string.lua", true)
+        test_rtp_file("plugin/file_drv_string.lua", true)
         test_rtp_file("plugin/test.vim", false)
         test_rtp_file("plugin/test.json", false)
+        test_rtp_file("test.txt", false)
 
-        -- files
-        test_rtp_file("plugin/file.lua", false)
+        -- lua files are byte compiled
+        test_rtp_file("plugin/file.lua", true)
         test_rtp_file("plugin/file.vim", false)
 
         -- Plugins and neovim runtime aren't byte compiled by default
@@ -136,4 +162,24 @@ in
       assert(not is_byte_compiled(init), "MYVIMRC is not expected to be byte compiled, but it is")
     '';
   };
+
+  configs-disabled = {
+    performance.byteCompileLua = {
+      enable = true;
+      configs = false;
+    };
+
+    extraFiles."plugin/test1.lua".text = "vim.opt.tabstop = 2";
+
+    files."plugin/test2.lua".opts.tabstop = 2;
+
+    extraConfigLuaPost = ''
+      ${isByteCompiledFun}
+
+      -- extraFiles
+      test_rtp_file("plugin/test1.lua", false)
+      -- files
+      test_rtp_file("plugin/test2.lua", false)
+    '';
+  };
 }

tests/test-sources/modules/performance/files/file.lua

@@ -0,0 +1 @@
+vim.opt.tabstop = 2

wrappers/_shared.nix

@@ -53,11 +53,11 @@ in
         setAttrByPath filesOpt (
           listToAttrs (
             map (
-              { target, source, ... }:
+              { target, finalSource, ... }:
               {
                 name = filesPrefix + target;
                 value = {
-                  inherit source;
+                  source = finalSource;
                 };
               }
             ) extraFiles