Merge branch 'feat/importing-nixlets' into 'main'

feat: Nixlet Dependencies/importing Nixlets

See merge request TECHNOFAB/nixlets!7

docs/generating_docs.md

@@ -11,6 +11,7 @@ This is all that's needed:
 ```nix
 (<nixlet>).mkDocs {
 # Params:
+#  fullValues ? false,
 #  transformOptions ? opt: opt,
 #  filter ? _: true,
 #  headingDepth ? 3,
@@ -36,3 +37,18 @@ string
 "Hello world!"
 ```
 ````
+
+The `fullValues` param controls whether the docs should include dependency Nixlets.
+For example, when defining `postgres` as a dependency, by default the docs would not
+include these options. If it's `true`, everything is included.
+
+Dependency Nixlets' options which you override from your own `values.nix` will show both
+default values:
+
+````md
+**Overridden value**:
+
+```nix
+<the overridden value set in values.nix>
+```
+````

docs/importing.md

@@ -0,0 +1,27 @@
+# Importing Nixlets
+
+Nixlets can now define dependency Nixlets and handle them similarly to how nested
+Helm Charts work.
+
+## Importing
+
+To define a dependency Nixlet, give it a name and pass the Nixlet as a value:
+
+```nix title="default.nix of Nixlet"
+  nixlet.dependencies."postgres" = <any nixlet>;
+```
+
+`<any nixlet>` here could be stuff like `nixlet-lib.fetchNixletFromGitlab {...}`,
+`nixlet-lib.fetchNixlet <url> <sha>`, etc.
+
+## Defining Values
+
+You can pre-define values for dependency Nixlets like this:
+
+```nix title="values.nix of Nixlet"
+  options = {
+    # options for the current Nixlet
+  };
+  # overwriting the default of dependency Nixlets (the user can still overwrite this using mkForce for example)
+  config."postgres".replicaCount = 10;
+```

docs/options.md

@@ -0,0 +1,3 @@
+# Options
+
+{{ include_raw("options.md") }}

lib/default.nix

@@ -2,155 +2,227 @@
   lib,
   kubenix,
   ...
-} @ attrs:
+} @ attrs: let
-with lib; rec {
+  inherit (lib) mkOption types evalModules concatMapStringsSep assertMsg;
-  evalValues = file: {rawValues, ...} @ args: (lib.evalModules {
+  nixlet-lib = rec {
-    specialArgs = {
+    nixletModule = ./nixletModule.nix;
-      utils = import ./utils.nix attrs;
+
-    };
+    evalValues = file: {
-    modules = [
+      rawValues,
-      file
+      dependencies,
-      (_: {
+      args,
-        # pass through all args to the values.nix module
+      check ? true,
-        config =
+      ...
-          rawValues
+    }: let
-          // {
+      moduleArgs =
-            _module.args = args;
+        args
+        // {
+          utils = import ./utils.nix attrs;
+        };
+      # get the values from the dependencies, then import them nested
+      # (so you can set postgres.replicaCount in values.nix for example when adding "postgres" as dependency)
+      extraModules = map (depName: {
+        options.${depName} = mkOption {
+          type = types.submodule {
+            imports = ["${dependencies.${depName}.path}/values.nix"];
+            _module.args =
+              moduleArgs
+              // {
+                # make sure that dependencies see their own name and version etc.
+                nixlet = {
+                  inherit (dependencies.${depName}) name version description;
+                  inherit (moduleArgs.nixlet) project;
+                };
+              };
           };
-      })
+          default = {};
-    ];
+          description = let
-  });
+            n = dependencies.${depName};
-  mkValues = file: args: (evalValues file args).config;
+          in ''
+            Imported Nixlet as a dependency:
 
-  # wraps mkNixletInner to allow passing either a path or an attrset
+            |Name|Version|Description|
-  mkNixlet = arg:
+            |----|-------|-----------|
-    mkNixletInner (
+            |${n.name}|${n.version}|${n.description}|
-      if (builtins.typeOf arg) == "set"
+          '';
-      then arg
+        };
-      else
+      }) (builtins.attrNames dependencies);
-        {path = arg;}
+    in
-        // (
+      builtins.addErrorContext "[nixlets] while evaluating values" (
-          if builtins.pathExists "${arg}/nixlet.nix"
+        evalModules {
-          then (import "${arg}/nixlet.nix")
+          modules =
-          else throw "Nixlet at '${arg}' does not contain nixlet.nix and mkNixlet was called with just a path"
+            [
-        )
+              file
-    );
+              {
+                _module = {
+                  args = moduleArgs;
+                  inherit check;
+                };
+              }
+              {config = rawValues;}
+            ]
+            ++ extraModules;
+        }
+      );
 
-  mkNixletInner = {
+    # wraps mkNixletInner to allow passing either a path or an attrset
-    path,
+    mkNixlet = arg:
-    name,
+      mkNixletInner (
-    version ? null,
+        if (builtins.typeOf arg) == "set"
-    description ? "",
+        then arg
-    defaultProject ? null,
+        else
-    ...
+          {path = arg;}
-  }: let
+          // (
-    # every nixlet gets "nixlet" as arg with some useful data about itself
+            if builtins.pathExists "${arg}/nixlet.nix"
-    baseNixletArg = {
+            then (import "${arg}/nixlet.nix")
-      inherit name version description;
+            else throw "Nixlet at '${arg}' does not contain nixlet.nix and mkNixlet was called with just a path"
-      project = defaultProject;
+          )
-    };
+      );
-    nixlet = {
+
-      inherit name version description path;
+    mkNixletInner = {
-      values = evalValues "${path}/values.nix" {
+      path,
-        rawValues = {};
+      name,
-        nixlet = baseNixletArg;
+      version ? null,
+      description ? "",
+      defaultProject ? null,
+      ...
+    }: let
+      # every nixlet gets "nixlet" as arg with some useful data about itself
+      baseNixletArg = {
+        inherit name version description;
+        project = defaultProject;
       };
-      mkDocs = opts: mkDocs (opts // {inherit nixlet;});
+      nixlet = {
-      eval = {
+        _type = "nixlet";
-        system,
+        inherit name version description path;
-        project ? defaultProject,
+        # just values of the current nixlet (lighweight)
-        overrides ? (_: {}),
+        values = evalValues "${path}/values.nix" {
-        values ? {},
+          rawValues = {};
-      }:
+          dependencies = {};
-        assert lib.assertMsg (project != null) "No default project set, please pass a project to the render method"; let
+          # no checking since this doesn't include dependencies
-          nixletArg = baseNixletArg // {inherit project;};
+          check = false;
+          args.nixlet = baseNixletArg;
+        };
+        # full values, including dependencies etc. (complex)
+        fullValues = args: let
+          evaled = nixlet.eval args;
         in
-          kubenix.evalModules.${system} {
+          evalValues "${path}/values.nix" {
-            module = {kubenix, ...}: {
+            rawValues = {};
-              imports = with kubenix.modules; [
+            inherit (evaled.config.nixlet) dependencies;
-                k8s
+            args.nixlet = baseNixletArg;
-                helm
-                docker
-                files
-                ./secretsModule.nix
-                (_: let
-                  finalValues = mkValues "${path}/values.nix" {
-                    rawValues = values;
-                    nixlet = nixletArg;
-                  };
-                in {
-                  imports = [path];
-                  _module.args.nixlet =
-                    {
-                      values = finalValues;
-                    }
-                    // nixletArg;
-                })
-                overrides
-              ];
-              kubenix.project = project;
-            };
           };
-      render = {
+        mkDocs = opts: mkDocs (opts // {inherit nixlet;});
-        system,
+        eval = {
-        project ? defaultProject,
+          system,
-        overrides ? (_: {}),
+          project ? defaultProject,
-        values ? {},
+          overrides ? (_: {}),
-      }:
+          values ? {},
-        (nixlet.eval {
+        }:
-          inherit system project overrides values;
+          assert assertMsg (project != null) "No default project set, please pass a project to the eval/render method"; let
-        })
+            nixletArg = baseNixletArg // {inherit project;};
-        .config
+          in
-        .kubernetes
+            builtins.addErrorContext "[nixlets] while evaluating nixlet ${name}" (
-        .resultYAML;
+              kubenix.evalModules.${system} {
-      # combines all secrets files in a single directory
+                module = {
-      secrets = args: (nixlet.eval args).config.kubernetes.secretsCombined;
+                  config,
-    };
+                  kubenix,
-  in
+                  ...
-    nixlet;
+                }: {
+                  imports = with kubenix.modules; [
+                    k8s
+                    helm
+                    docker
+                    files
+                    ./secretsModule.nix
+                    ./nixletModule.nix
+                    (let
+                      finalValues =
+                        (evalValues "${path}/values.nix" {
+                          rawValues = values;
+                          inherit (config.nixlet) dependencies;
+                          args.nixlet = nixletArg;
+                        }).config;
+                    in {
+                      imports = [path];
+                      _module.args = {
+                        nixlet =
+                          {
+                            values = finalValues;
+                          }
+                          // nixletArg;
+                        inherit nixlet-lib system;
+                      };
+                    })
+                    overrides
+                  ];
+                  kubenix.project = project;
+                };
+              }
+            );
+        render = {
+          system,
+          project ? defaultProject,
+          overrides ? (_: {}),
+          values ? {},
+        }:
+          (nixlet.eval {
+            inherit system project overrides values;
+          })
+          .config
+          .kubernetes
+          .resultYAML;
+        # combines all secrets files in a single directory
+        secrets = args: (nixlet.eval args).config.kubernetes.secretsCombined;
 
-  fetchNixlet = url: sha256: mkNixlet (builtins.fetchTarball {inherit url sha256;});
+      };
-  fetchNixletFromGitlab = {
+    in
-    project,
+      nixlet;
-    name,
-    version,
-    sha256,
-  }: let
-    projectEscaped = builtins.replaceStrings ["/"] ["%2F"] project;
-  in
-    fetchNixlet "https://gitlab.com/api/v4/projects/${projectEscaped}/packages/generic/${name}/${version}/${name}.tar.gz" sha256;
 
-  uploadNixletsToGitlab = {
+    fetchNixlet = url: sha256: mkNixlet (builtins.fetchTarball {inherit url sha256;});
-    pkgs,
+    fetchNixletFromGitlab = {
-    projectId,
+      project,
-    nixlets,
+      name,
-    ...
+      version,
-  }:
+      sha256,
-    pkgs.writeShellScriptBin "nixlets-upload" (
+    }: let
-      ''
+      projectEscaped = builtins.replaceStrings ["/"] ["%2F"] project;
-        if [[ -z "$AUTH_HEADER" ]]; then
+    in
-          echo "Must provide AUTH_HEADER environment variable!" 1>&2
+      fetchNixlet "https://gitlab.com/api/v4/projects/${projectEscaped}/packages/generic/${name}/${version}/${name}.tar.gz" sha256;
-          exit 1
-        fi
-      ''
-      + lib.concatStringsSep "\n" (
-        builtins.map (nixlet:
-          with nixlet; ''
-            URL="https://gitlab.com/api/v4/projects/${projectId}/packages/generic/${name}/${version}/${name}.tar.gz"
-            if ${pkgs.curl}/bin/curl --output /dev/null --silent --head --fail --header "$AUTH_HEADER" $URL; then
-              echo "> Skipped ${name}@${version} because it already exists in the Package Registry"
-            else
-              echo "> Uploading new version ${name}@${version}"
-              ${pkgs.gnutar}/bin/tar -czf /tmp/${name}.tar.gz --mode='u+rwX' -C ${path} --transform 's/^\./\/${name}/' .
-              ${pkgs.curl}/bin/curl --header "$AUTH_HEADER" --upload-file "/tmp/${name}.tar.gz" "$URL"; echo;
-              ${pkgs.coreutils}/bin/rm -f /tmp/${nixlet.name}.tar.gz
-              echo "> Finished ${name}@${version}, see above"
-            fi
-          '')
-        nixlets
-      )
-    );
 
-  mkDocs = opts:
+    uploadNixletsToGitlab = {
-    import ./valuesDocs.nix (opts // {inherit lib;});
+      pkgs,
-}
+      projectId,
+      nixlets,
+      ...
+    }:
+      pkgs.writeShellScriptBin "nixlets-upload" (
+        ''
+          if [[ -z "$AUTH_HEADER" ]]; then
+            echo "Must provide AUTH_HEADER environment variable!" 1>&2
+            exit 1
+          fi
+        ''
+        + concatMapStringsSep "\n" (
+          (nixlet:
+            with nixlet; ''
+              URL="https://gitlab.com/api/v4/projects/${projectId}/packages/generic/${name}/${version}/${name}.tar.gz"
+              if ${pkgs.curl}/bin/curl --output /dev/null --silent --head --fail --header "$AUTH_HEADER" $URL; then
+                echo "> Skipped ${name}@${version} because it already exists in the Package Registry"
+              else
+                echo "> Uploading new version ${name}@${version}"
+                ${pkgs.gnutar}/bin/tar -czf /tmp/${name}.tar.gz --mode='u+rwX' -C ${path} --transform 's/^\./\/${name}/' .
+                ${pkgs.curl}/bin/curl --header "$AUTH_HEADER" --upload-file "/tmp/${name}.tar.gz" "$URL"; echo;
+                ${pkgs.coreutils}/bin/rm -f /tmp/${nixlet.name}.tar.gz
+                echo "> Finished ${name}@${version}, see above"
+              fi
+            '')
+          nixlets
+        )
+      );
+
+    mkDocs = opts:
+      import ./valuesDocs.nix (opts // {inherit lib;});
+  };
+in
+  nixlet-lib

lib/nixletModule.nix

@@ -0,0 +1,75 @@
+{
+  lib,
+  config,
+  nixlet,
+  system,
+  ...
+}: let
+  inherit (lib) mkOption types mkOptionType isType mkMerge mapAttrs mkIf literalExpression;
+  cfg = config.nixlet;
+
+  nixletType = mkOptionType {
+    name = "nixlet";
+    description = "reference";
+    descriptionClass = "noun";
+    check = isType "nixlet";
+  };
+in {
+  imports = [
+    {
+      # shortcut, allows accessing deps a bit shorter/more easily
+      _module.args.deps = cfg.deps;
+    }
+  ];
+  options.nixlet = {
+    dependencies = mkOption {
+      type = types.attrsOf nixletType;
+      default = {};
+      description = ''
+        Import other nixlets as dependencies. Works similar to Helm, specify values for these
+        Nixlets by using their name as a prefix. Like `postgres.replicaCount` in `values.nix` for example.
+      '';
+      example = literalExpression ''
+        {
+          "postgres" = nixlet-lib.mkNixlet <path>;
+          "mongodb" = nixlet-lib.fetchNixlet ...; # etc.
+        }
+      '';
+    };
+    deps = mkOption {
+      readOnly = true;
+      type = types.attrsOf types.attrs;
+      default = mapAttrs (name: val:
+        builtins.addErrorContext "[nixlets] while evaluating dependency ${name}"
+        (val.eval {
+          inherit system;
+          inherit (config.kubenix) project;
+          values = nixlet.values.${name};
+        }).config)
+      cfg.dependencies;
+      description = ''
+        Evaluated dependency nixlets. Allows accessing their resources like for example:
+
+        ```nix
+        config.nixlet.deps."<name>".kubernetes.resources
+        ```
+      '';
+    };
+    depAutoMerge = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Whether to automatically merge dependency nixlets' configs
+        with the current nixlet. If disabled, you can access dependency outputs via:
+
+        ```nix
+        config.nixlet.deps."<name>".kubernetes.resources
+        ```
+      '';
+    };
+  };
+
+  config = mkIf cfg.depAutoMerge {
+    kubernetes.resources = mkMerge (map (dep: dep.kubernetes.resources) (builtins.attrValues cfg.deps));
+  };
+}

lib/valuesDocs.nix

@@ -1,6 +1,8 @@
 {
   lib,
   nixlet,
+  # whether to generate docs for the full values, including dependencies
+  fullValues ? false,
   transformOptions ? opt: opt,
   filter ? _: true,
   headingDepth ? 3,
@@ -13,7 +15,12 @@
     mapAttrsToList
     concatStrings
     replicate
+    optionalString
+    optionAttrSetToDocList
+    attrByPath
+    generators
     ;
+  inherit (generators) toPretty;
 
   _transformOptions = opt:
     transformOptions (opt
@@ -25,7 +32,12 @@
         name = lib.removePrefix "config." opt.name;
       });
 
-  rawOpts = lib.optionAttrSetToDocList nixlet.values.options;
+  valueSource =
+    if fullValues
+    # TODO: get rid of system, just here cuz of kubenix
+    then (nixlet.fullValues {system = "x86_64-linux";})
+    else nixlet.values;
+  rawOpts = optionAttrSetToDocList valueSource.options;
   transformedOpts = map _transformOptions rawOpts;
   filteredOpts = lib.filter (opt: opt.visible && !opt.internal) transformedOpts;
 
@@ -58,7 +70,20 @@
       ${opt.type}
       ```
     ''
-    + (lib.optionalString (opt ? default && opt.default != null) ''
+    # used to show what changes a nixlet did to values of dependencies
+    + (let
+      val = toPretty {} (attrByPath opt.loc "_not found_" valueSource.config);
+      default = removeSuffix "\n" opt.default.text;
+    in
+      optionalString (opt.type != "submodule" && val != default)
+      ''
+        **Overridden value**:
+
+        ```nix
+        ${val}
+        ```
+      '')
+    + (optionalString (opt ? default && opt.default != null) ''
 
       **Default value**:
 
@@ -66,7 +91,7 @@
       ${removeSuffix "\n" opt.default.text}
       ```
     '')
-    + (lib.optionalString (opt ? example) ''
+    + (optionalString (opt ? example) ''
 
       **Example value**:
 

nix/repo/docs.nix

@@ -3,8 +3,22 @@
   cell,
   ...
 }: let
-  inherit (inputs) doclib;
+  inherit (inputs) pkgs doclib nixlet-lib;
   inherit (cell) nixlets;
+
+  optionsDoc = doclib.mkOptionDocs {
+    module = nixlet-lib.nixletModule;
+    roots = [
+      {
+        url = "https://gitlab.com/TECHNOFAB/nixlets/-/blob/main/lib";
+        path = "${inputs.self}/lib";
+      }
+    ];
+  };
+  optionsDocs = pkgs.runCommand "options-docs" {} ''
+    mkdir -p $out
+    ln -s ${optionsDoc} $out/options.md
+  '';
 in
   (doclib.mkDocs {
     docs."default" = {
@@ -23,9 +37,13 @@ in
           domains = ["nixlets.projects.tf"];
         };
       };
+      macros = {
+        enable = true;
+        includeDir = toString optionsDocs;
+      };
       dynamic-nav = {
         enable = true;
-        files."Nixlets Values" = builtins.map (val: {${val.name} = val.mkDocs {};}) (builtins.attrValues nixlets);
+        files."Nixlets Values" = builtins.map (val: {${val.name} = val.mkDocs {fullValues = true;};}) (builtins.attrValues nixlets);
       };
       config = {
         site_name = "Nixlets";
@@ -43,8 +61,10 @@ in
           {"Creating Nixlets" = "creation.md";}
           {"Packaging" = "packaging.md";}
           {"Usage" = "usage.md";}
+          {"Importing" = "importing.md";}
           {"Generating Docs" = "generating_docs.md";}
           {"Secrets" = "secrets.md";}
+          {"Options" = "options.md";}
         ];
         markdown_extensions = [
           {

tests/fixtures/dependency/default.nix

@@ -0,0 +1,3 @@
+{nixlet-lib, ...}: {
+  config.nixlet.dependencies."example" = nixlet-lib.mkNixlet ../example;
+}

tests/fixtures/dependency/nixlet.nix

@@ -0,0 +1,6 @@
+{
+  name = "dep";
+  version = "0.0.1";
+  description = "hello world";
+  defaultProject = "dep";
+}

tests/fixtures/dependency/values.nix

@@ -0,0 +1,5 @@
+_: {
+  options = {};
+
+  config."example".example = "Hello dependency!";
+}

tests/fixtures/example/default.nix

@@ -0,0 +1,3 @@
+{nixlet, ...}: with nixlet; {
+  kubernetes.resources.configMaps."test".data."test" = values.example;
+}

tests/lib_test.nix

@@ -3,11 +3,14 @@
   ntlib,
   nixlet-lib,
   ...
-}: {
+}: let
+  inherit (pkgs.lib) mkForce;
+in {
   suites."Lib Tests" = {
     pos = __curPos;
     tests = let
       nixlet = nixlet-lib.mkNixlet ./fixtures/example;
+      depNixlet = nixlet-lib.mkNixlet ./fixtures/dependency;
     in [
       {
         name = "mkNixlet fail on nonexistant nixlet.nix";
@@ -42,6 +45,27 @@
             assert_file_contains "${docs}" '"Hello world!"'
           '';
       }
+      {
+        name = "Nixlet dependencies";
+        expected = "Hello dependency!";
+        actual = let
+          evaled = depNixlet.eval {inherit (pkgs.stdenv.hostPlatform) system;};
+        in
+          evaled.config.kubernetes.resources.configMaps."test".data."test";
+      }
+      {
+        name = "Nixlet dependency value override";
+        expected = "Hello override!";
+        actual = let
+          evaled = depNixlet.eval {
+            inherit (pkgs.stdenv.hostPlatform) system;
+            values = {
+              "example".example = mkForce "Hello override!";
+            };
+          };
+        in
+          evaled.config.kubernetes.resources.configMaps."test".data."test";
+      }
     ];
   };
 }