docs: Correctly generate documentation for types either a (submodule ...) (#929)

This requires pulling quite a lot of code from nixpkgs in order to
make the `either` type work correctly, the effects can be seen for
example in the documentation of `ollama.promps` and `ollama.actions`
This commit is contained in:
traxys 2024-01-12 23:22:03 +01:00 committed by GitHub
parent 61da581315
commit 29225c2797
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 509 additions and 308 deletions

View file

@ -1,262 +1,255 @@
{ {
pkgs,
lib, lib,
modules, modules,
... pkgs,
}: }: let
with lib; let nixvimPath = toString ./..;
options = lib.evalModules {
inherit modules; gitHubDeclaration = user: repo: subpath: {
specialArgs = {inherit pkgs lib;}; url = "https://github.com/${user}/${repo}/blob/master/${subpath}";
name = "<${repo}/${subpath}>";
}; };
mkMDDoc = options: transformOptions = opt:
(pkgs.nixosOptionsDoc { opt
options = filterAttrs (k: _: k != "_module") options;
warningsAreErrors = false;
transformOptions = opts:
opts
// { // {
declarations = with builtins; declarations =
map map (
(
decl: decl:
if hasPrefix "/nix/store/" decl if pkgs.lib.hasPrefix nixvimPath (toString decl)
then let then
filepath = toString (match "^/nix/store/[^/]*/(.*)" decl); gitHubDeclaration "nix-community" "nixvim"
in { (pkgs.lib.removePrefix "/" (pkgs.lib.removePrefix nixvimPath (toString decl)))
url = "https://github.com/nix-community/nixvim/blob/main/${filepath}"; else if decl == "lib/modules.nix"
name = filepath; then gitHubDeclaration "NixOS" "nixpkgs" decl
}
else decl else decl
) )
opts.declarations; opt.declarations;
}; };
})
.optionsCommonMark;
removeUnwanted = attrs: getSubOptions' = type: loc: let
builtins.removeAttrs attrs [ types =
"_module" # Composite types
"_freeformOptions" {
"warnings" either =
"assertions" (getSubOptions' type.nestedTypes.left loc)
"content" // (getSubOptions' type.nestedTypes.right loc);
]; nullOr = getSubOptions' type.nestedTypes.elemType loc;
lazyAttrsOf = getSubOptions' type.nestedTypes.elemType (loc ++ ["<name>"]);
attrsOf = getSubOptions' type.nestedTypes.elemType (loc ++ ["<name>"]);
listOf = getSubOptions' type.nestedTypes.elemType (loc ++ ["*"]);
functionTo = getSubOptions' type.nestedTypes.elemType (loc ++ ["<function body>"]);
removeWhitespace = builtins.replaceStrings [" "] [""]; # Taken from lib/types.nix
submodule = let
getSubOptions = opts: path: removeUnwanted (opts.type.getSubOptions path); base = lib.evalModules {
modules =
isVisible = opts: [
if isOption opts {
then attrByPath ["visible"] true opts _module.args.name = lib.mkOptionDefault "name";
else if opts.isOption }
then attrByPath ["index" "options" "visible"] true opts ]
else let ++ type.getSubModules;
filterFunc = };
filterAttrs inherit (base._module) freeformType;
(
_: v:
if isAttrs v
then isVisible v
else true
);
hasEmptyIndex = (filterFunc opts.index.options) == {};
hasEmptyComponents = (filterFunc opts.components) == {};
in in
!hasEmptyIndex || !hasEmptyComponents; (base.extendModules
{prefix = loc;})
wrapModule = path: opts: isOpt: rec { .options
index = { // lib.optionalAttrs (freeformType != null) {
options = _freeformOptions = getSubOptions' freeformType loc;
if isOpt
then opts
else filterAttrs (_: component: component.isOption && (isVisible component)) opts;
path = removeWhitespace (concatStringsSep "/" path);
}; };
}
components = # Leaf types
if isOpt // lib.genAttrs [
"raw"
"bool"
"optionType"
"unspecified"
"str"
"attrs"
"rawLua"
"int"
"package"
"numberBetween"
"enum"
"anything"
"separatedString"
"path"
"maintainer"
"unsignedInt"
"float"
"positiveInt"
"intBetween"
"nullType"
"nonEmptyStr"
] (_: {});
in
# For recursive types avoid calculating sub options, else this
# will end up in an unbounded recursion
if loc == ["plugins" "packer" "plugins" "*" "requires"]
then {} then {}
else filterAttrs (_: component: !component.isOption && (isVisible component)) opts; else if builtins.hasAttr type.name types
then types.${type.name}
else throw "unhandled type in documentation: ${type.name}";
hasComponents = components != {}; mkOptionsJSON = options: let
# Mainly present to patch the type.getSubOptions of `either`, but we need to patch all
isOption = isOpt; # the options in order to correctly handle other composite options
}; # The code that follows is taken almost exactly from nixpkgs,
# by changing type.getSubOptions to getSubOptions'
processModulesRec = modules: let # lib/options.nix
recurse = path: mods: let optionAttrSetToDocList' = _: options:
g = name: opts: lib.concatMap (opt: let
if !isOption opts name = lib.showOption opt.loc;
then wrapModule (path ++ [name]) (recurse (path ++ [name]) opts) false docOption =
else let {
subOpts = getSubOptions opts (path ++ [name]); inherit (opt) loc;
in inherit name;
if subOpts != {} description = opt.description or null;
then declarations = builtins.filter (x: x != lib.unknownModule) opt.declarations;
wrapModule internal = opt.internal or false;
(path ++ [name]) visible =
( if (opt ? visible && opt.visible == "shallow")
(recurse (path ++ [name]) subOpts) then true
// { else opt.visible or true;
# This is necessary to include the submodule option's definition in the docs (description, type, etc.) readOnly = opt.readOnly or false;
# For instance, this helps submodules like "autoCmd" to include their base definitions and examples in the docs type = opt.type.description or "unspecified";
# Though there might be a better, less "hacky" solution than this.
${name} = recursiveUpdate opts {
isOption = true;
type.getSubOptions = _: _: {}; # Used to exclude suboptions from the submodule definition itself
};
} }
) // lib.optionalAttrs (opt ? example) {
false example = builtins.addErrorContext "while evaluating the example of option `${name}`" (
else wrapModule (path ++ [name]) opts true; lib.options.renderOptionValue opt.example
in
mapAttrs g mods;
in
foldlAttrs
(
acc: name: opts: let
group =
if !opts.hasComponents
then "Neovim Options"
else "none";
last =
acc.${group}
or {
index = {
options = {};
path = removeWhitespace "${group}";
};
components = {};
isGroup = true;
hasComponents = false;
};
isOpt = !opts.hasComponents && (isOption opts.index.options);
in
acc
// {
${group} = recursiveUpdate last {
index.options = optionalAttrs isOpt {
${name} = opts.index.options;
};
components = optionalAttrs (!isOpt) {
${name} = recursiveUpdate opts {
index.path =
removeWhitespace
(
concatStringsSep "/"
(
(optional (group != "none") group) ++ [opts.index.path]
)
); );
hasComponents = true;
};
};
hasComponents = last.components != {};
};
} }
) // lib.optionalAttrs (opt ? defaultText || opt ? default) {
{} default =
(recurse [] modules); builtins.addErrorContext "while evaluating the ${
if opt ? defaultText
then "defaultText"
else "default value"
} of option `${name}`" (
lib.options.renderOptionValue (opt.defaultText or opt.default)
);
}
// lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != null) {inherit (opt) relatedPackages;};
mapModulesToString = f: modules: let subOptions = let
recurse = mods: let ss = getSubOptions' opt.type opt.loc;
g = name: opts:
if (opts ? "isGroup")
then
if name != "none"
then (f name opts) + ("\n" + recurse opts.components)
else recurse opts.components
else if opts.hasComponents
then (f name opts) + ("\n" + recurse opts.components)
else f name opts;
in in
concatStringsSep "\n" (mapAttrsToList g mods); if ss != {}
then optionAttrSetToDocList' opt.loc ss
else [];
subOptionsVisible = docOption.visible && opt.visible or null != "shallow";
in in
recurse modules; # To find infinite recursion in NixOS option docs:
# builtins.trace opt.loc
[docOption] ++ lib.optionals subOptionsVisible subOptions) (lib.collect lib.isOption options);
docs = rec { # Generate documentation template from the list of option declaration like
modules = processModulesRec (removeUnwanted options.options); # the set generated with filterOptionSets.
commands = optionAttrSetToDocList = optionAttrSetToDocList' [];
mapModulesToString
(
name: opts: let
isBranch =
if (hasSuffix "index" opts.index.path)
then true
else opts.hasComponents;
path =
if isBranch
then "${opts.index.path}/index.md"
else "${opts.index.path}.md";
in
(optionalString isBranch
"mkdir -p ${opts.index.path}\n")
+ "cp ${mkMDDoc opts.index.options} ${path}"
)
modules;
};
mdbook = { # nixos/lib/make-options-doc/default.nix
nixvimOptions =
mapModulesToString
(
name: opts: let
isBranch =
if name == "index"
then true
else opts.hasComponents && opts.index.options != {};
path = rawOpts = optionAttrSetToDocList options;
if isBranch transformedOpts = map transformOptions rawOpts;
then "${opts.index.path}/index.md" filteredOpts = lib.filter (opt: opt.visible && !opt.internal) transformedOpts;
else if !opts.hasComponents
then "${opts.index.path}.md"
else "";
indentLevel = with builtins; length (filter isString (split "/" opts.index.path)) - 1; # Generate DocBook documentation for a list of packages. This is
# what `relatedPackages` option of `mkOption` from
padding = concatStrings (builtins.genList (_: "\t") indentLevel); # ../../../lib/options.nix influences.
in "${padding}- [${name}](${path})" #
) # Each element of `relatedPackages` can be either
docs.modules; # - a string: that will be interpreted as an attribute name from `pkgs` and turned into a link
}; # to search.nixos.org,
# - a list: that will be interpreted as an attribute path from `pkgs` and turned into a link
prepareMD = '' # to search.nixos.org,
# Copy inputs into the build directory # - an attrset: that can specify `name`, `path`, `comment`
cp -r --no-preserve=all $inputs/* ./ # (either of `name`, `path` is required, the rest are optional).
cp ${../CONTRIBUTING.md} ./CONTRIBUTING.md #
# NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists.
# Copy the generated MD docs into the build directory # Such checks are not compatible with option docs caching.
# Using pkgs.writeShellScript helps to avoid the "bash: argument list too long" error genRelatedPackages = packages: optName: let
bash -e ${pkgs.writeShellScript "copy_docs" docs.commands} unpack = p:
if lib.isString p
# Prepare SUMMARY.md for mdBook then {name = p;}
# Using pkgs.writeText helps to avoid the same error as above else if lib.isList p
substituteInPlace ./SUMMARY.md \ then {path = p;}
--replace "@NIXVIM_OPTIONS@" "$(cat ${pkgs.writeText "nixvim-options-summary.md" mdbook.nixvimOptions})" else p;
describe = args: let
title = args.title or null;
name = args.name or (lib.concatStringsSep "." args.path);
in ''
- [${lib.optionalString (title != null) "${title} aka "}`pkgs.${name}`](
https://search.nixos.org/packages?show=${name}&sort=relevance&query=${name}
)${
lib.optionalString (args ? comment) "\n\n ${args.comment}"
}
''; '';
in in
pkgs.stdenv.mkDerivation { lib.concatMapStrings (p: describe (unpack p)) packages;
name = "nixvim-docs";
phases = ["buildPhase"]; nixvimOptionsList =
lib.flip map filteredOpts
buildInputs = [pkgs.mdbook]; (
inputs = sourceFilesBySuffices ./. [".md" ".toml" ".js"]; opt:
opt
buildPhase = '' // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != []) {
dest=$out/share/doc relatedPackages = genRelatedPackages opt.relatedPackages opt.name;
mkdir -p $dest }
${prepareMD} );
mdbook build
cp -r ./book/* $dest nixvimOptionsNix = builtins.listToAttrs (map (o: {
''; inherit (o) name;
value = removeAttrs o ["name" "visible" "internal"];
})
nixvimOptionsList);
in
pkgs.runCommand "options.json"
{
meta.description = "List of NixOS options in JSON format";
nativeBuildInputs = [
pkgs.brotli
pkgs.python3Minimal
];
options =
builtins.toFile "options.json"
(builtins.unsafeDiscardStringContext (builtins.toJSON nixvimOptionsNix));
baseJSON = builtins.toFile "base.json" "{}";
}
''
# Export list of options in different format.
dst=$out/share/doc/nixos
mkdir -p $dst
TOUCH_IF_DB=$dst/.used-docbook \
python ${pkgs.path}/nixos/lib/make-options-doc/mergeJSON.py \
$baseJSON $options \
> $dst/options.json
if grep /nixpkgs/nixos/modules $dst/options.json; then
echo "The manual appears to depend on the location of Nixpkgs, which is bad"
echo "since this prevents sharing via the NixOS channel. This is typically"
echo "caused by an option default that refers to a relative path (see above"
echo "for hints about the offending path)."
exit 1
fi
brotli -9 < $dst/options.json > $dst/options.json.br
mkdir -p $out/nix-support
echo "file json $dst/options.json" >> $out/nix-support/hydra-build-products
echo "file json-br $dst/options.json.br" >> $out/nix-support/hydra-build-products
'';
in
rec {
options-json = mkOptionsJSON (lib.evalModules {inherit modules;}).options;
man-docs = pkgs.callPackage ./man {inherit options-json;};
}
# Do not check if documentation builds fine on darwin as it fails:
# > sandbox-exec: pattern serialization length 69298 exceeds maximum (65535)
// lib.optionalAttrs (!pkgs.stdenv.isDarwin) {
docs = pkgs.callPackage ./mdbook {
inherit mkOptionsJSON modules getSubOptions';
};
} }

19
docs/man/default.nix Normal file
View file

@ -0,0 +1,19 @@
{
options-json,
runCommand,
installShellFiles,
nixos-render-docs,
}:
runCommand "nixvim-configuration-reference-manpage" {
nativeBuildInputs = [installShellFiles nixos-render-docs];
} ''
# Generate man-pages
mkdir -p $out/share/man/man5
nixos-render-docs -j $NIX_BUILD_CORES options manpage \
--revision unstable \
--header ${./nixvim-header.5} \
--footer ${./nixvim-footer.5} \
${options-json}/share/doc/nixos/options.json \
$out/share/man/man5/nixvim.5
compressManPages $out
''

253
docs/mdbook/default.nix Normal file
View file

@ -0,0 +1,253 @@
{
pkgs,
lib,
modules,
mkOptionsJSON,
runCommand,
nixos-render-docs,
getSubOptions',
}:
with lib; let
options = lib.evalModules {
inherit modules;
specialArgs = {inherit pkgs lib;};
};
mkMDDoc = options: let
optionsJson = mkOptionsJSON options;
in
runCommand "options.md" {
nativeBuildInputs = [nixos-render-docs];
} ''
nixos-render-docs -j $NIX_BUILD_CORES options commonmark \
--manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \
--revision "" \
${optionsJson}/share/doc/nixos/options.json \
$out
'';
removeUnwanted = attrs:
builtins.removeAttrs attrs [
"_module"
"_freeformOptions"
"warnings"
"assertions"
"content"
];
removeWhitespace = builtins.replaceStrings [" "] [""];
getSubOptions = opts: path: removeUnwanted (getSubOptions' opts.type path);
isVisible = opts:
if isOption opts
then attrByPath ["visible"] true opts
else if opts.isOption
then attrByPath ["index" "options" "visible"] true opts
else let
filterFunc =
filterAttrs
(
_: v:
if isAttrs v
then isVisible v
else true
);
hasEmptyIndex = (filterFunc opts.index.options) == {};
hasEmptyComponents = (filterFunc opts.components) == {};
in
!hasEmptyIndex || !hasEmptyComponents;
wrapModule = path: opts: isOpt: rec {
index = {
options =
if isOpt
then opts
else filterAttrs (_: component: component.isOption && (isVisible component)) opts;
path = removeWhitespace (concatStringsSep "/" path);
};
components =
if isOpt
then {}
else filterAttrs (_: component: !component.isOption && (isVisible component)) opts;
hasComponents = components != {};
isOption = isOpt;
};
processModulesRec = modules: let
recurse = path: mods: let
g = name: opts:
if !isOption opts
then wrapModule (path ++ [name]) (recurse (path ++ [name]) opts) false
else let
subOpts = getSubOptions opts (path ++ [name]);
in
if subOpts != {}
then
wrapModule
(path ++ [name])
(
(recurse (path ++ [name]) subOpts)
// {
# This is necessary to include the submodule option's definition in the docs (description, type, etc.)
# For instance, this helps submodules like "autoCmd" to include their base definitions and examples in the docs
# Though there might be a better, less "hacky" solution than this.
${name} = recursiveUpdate opts {
isOption = true;
type.getSubOptions = _: _: {}; # Used to exclude suboptions from the submodule definition itself
};
}
)
false
else wrapModule (path ++ [name]) opts true;
in
mapAttrs g mods;
in
foldlAttrs
(
acc: name: opts: let
group =
if !opts.hasComponents
then "Neovim Options"
else "none";
last =
acc.${group}
or {
index = {
options = {};
path = removeWhitespace "${group}";
};
components = {};
isGroup = true;
hasComponents = false;
};
isOpt = !opts.hasComponents && (isOption opts.index.options);
in
acc
// {
${group} = recursiveUpdate last {
index.options = optionalAttrs isOpt {
${name} = opts.index.options;
};
components = optionalAttrs (!isOpt) {
${name} = recursiveUpdate opts {
index.path =
removeWhitespace
(
concatStringsSep "/"
(
(optional (group != "none") group) ++ [opts.index.path]
)
);
hasComponents = true;
};
};
hasComponents = last.components != {};
};
}
)
{}
(recurse [] modules);
mapModulesToString = f: modules: let
recurse = mods: let
g = name: opts:
if (opts ? "isGroup")
then
if name != "none"
then (f name opts) + ("\n" + recurse opts.components)
else recurse opts.components
else if opts.hasComponents
then (f name opts) + ("\n" + recurse opts.components)
else f name opts;
in
concatStringsSep "\n" (mapAttrsToList g mods);
in
recurse modules;
docs = rec {
modules = processModulesRec (removeUnwanted options.options);
commands =
mapModulesToString
(
name: opts: let
isBranch =
if (hasSuffix "index" opts.index.path)
then true
else opts.hasComponents;
path =
if isBranch
then "${opts.index.path}/index.md"
else "${opts.index.path}.md";
in
(optionalString isBranch
"mkdir -p ${opts.index.path}\n")
+ "cp ${mkMDDoc opts.index.options} ${path}"
)
modules;
};
mdbook = {
nixvimOptions =
mapModulesToString
(
name: opts: let
isBranch =
if name == "index"
then true
else opts.hasComponents && opts.index.options != {};
path =
if isBranch
then "${opts.index.path}/index.md"
else if !opts.hasComponents
then "${opts.index.path}.md"
else "";
indentLevel = with builtins; length (filter isString (split "/" opts.index.path)) - 1;
padding = concatStrings (builtins.genList (_: "\t") indentLevel);
in "${padding}- [${name}](${path})"
)
docs.modules;
};
prepareMD = ''
# Copy inputs into the build directory
cp -r --no-preserve=all $inputs/* ./
cp ${../../CONTRIBUTING.md} ./CONTRIBUTING.md
# Copy the generated MD docs into the build directory
# Using pkgs.writeShellScript helps to avoid the "bash: argument list too long" error
bash -e ${pkgs.writeShellScript "copy_docs" docs.commands}
# Prepare SUMMARY.md for mdBook
# Using pkgs.writeText helps to avoid the same error as above
substituteInPlace ./SUMMARY.md \
--replace "@NIXVIM_OPTIONS@" "$(cat ${pkgs.writeText "nixvim-options-summary.md" mdbook.nixvimOptions})"
'';
in
pkgs.stdenv.mkDerivation {
name = "nixvim-docs";
phases = ["buildPhase"];
buildInputs = [pkgs.mdbook];
inputs = sourceFilesBySuffices ./. [".md" ".toml" ".js"];
buildPhase = ''
dest=$out/share/doc
mkdir -p $dest
${prepareMD}
mdbook build
cp -r ./book/* $dest
'';
}

View file

@ -1,27 +1,14 @@
{ {
perSystem = { perSystem = {
pkgs, pkgs,
pkgsUnfree,
config, config,
modules, modules,
... ...
}: { }: {
packages = let packages = import ../docs {
# Do not check if documentation builds fine on darwin as it fails: inherit modules pkgs;
# > sandbox-exec: pattern serialization length 69298 exceeds maximum (65535) inherit (pkgs) lib;
docs = pkgs.lib.optionalAttrs (!pkgs.stdenv.isDarwin) {
docs = pkgsUnfree.callPackage (import ../docs) {
inherit modules;
}; };
};
man-docs = import ../man-docs {
pkgs = pkgsUnfree;
inherit modules;
};
in
docs
// man-docs;
# Test that all packages build fine when running `nix flake check`. # Test that all packages build fine when running `nix flake check`.
checks = config.packages; checks = config.packages;

View file

@ -1,51 +0,0 @@
{
pkgs,
modules,
}: let
nixvimPath = toString ./..;
gitHubDeclaration = user: repo: subpath: {
url = "https://github.com/${user}/${repo}/blob/master/${subpath}";
name = "<${repo}/${subpath}>";
};
in {
man-docs = let
eval = pkgs.lib.evalModules {
inherit modules;
};
options = pkgs.nixosOptionsDoc {
inherit (eval) options;
warningsAreErrors = false;
transformOptions = opt:
opt
// {
declarations =
map (
decl:
if pkgs.lib.hasPrefix nixvimPath (toString decl)
then
gitHubDeclaration "nix-community" "nixvim"
(pkgs.lib.removePrefix "/" (pkgs.lib.removePrefix nixvimPath (toString decl)))
else if decl == "lib/modules.nix"
then gitHubDeclaration "NixOS" "nixpkgs" decl
else decl
)
opt.declarations;
};
};
in
pkgs.runCommand "nixvim-configuration-reference-manpage" {
nativeBuildInputs = with pkgs; [installShellFiles nixos-render-docs];
} ''
# Generate man-pages
mkdir -p $out/share/man/man5
nixos-render-docs -j $NIX_BUILD_CORES options manpage \
--revision unstable \
--header ${./nixvim-header.5} \
--footer ${./nixvim-footer.5} \
${options.optionsJSON}/share/doc/nixos/options.json \
$out/share/man/man5/nixvim.5
compressManPages $out
'';
}