Merge branch 'feat/importing-nixlets' into 'main'

feat: Nixlet Dependencies/importing Nixlets

See merge request TECHNOFAB/nixlets!7

docs/generating_docs.md

@@ -11,6 +11,7 @@ This is all that's needed:
 ```nix
 (<nixlet>).mkDocs {
 # Params:
+#  fullValues ? false,
 #  transformOptions ? opt: opt,
 #  filter ? _: true,
 #  headingDepth ? 3,
@@ -36,3 +37,18 @@ string
 "Hello world!"
 ```
 ````
+
+The `fullValues` param controls whether the docs should include dependency Nixlets.
+For example, when defining `postgres` as a dependency, by default the docs would not
+include these options. If it's `true`, everything is included.
+
+Dependency Nixlets' options which you override from your own `values.nix` will show both
+default values:
+
+````md
+**Overridden value**:
+
+```nix
+<the overridden value set in values.nix>
+```
+````

docs/importing.md

@@ -0,0 +1,27 @@
+# Importing Nixlets
+
+Nixlets can now define dependency Nixlets and handle them similarly to how nested
+Helm Charts work.
+
+## Importing
+
+To define a dependency Nixlet, give it a name and pass the Nixlet as a value:
+
+```nix title="default.nix of Nixlet"
+  nixlet.dependencies."postgres" = <any nixlet>;
+```
+
+`<any nixlet>` here could be stuff like `nixlet-lib.fetchNixletFromGitlab {...}`,
+`nixlet-lib.fetchNixlet <url> <sha>`, etc.
+
+## Defining Values
+
+You can pre-define values for dependency Nixlets like this:
+
+```nix title="values.nix of Nixlet"
+  options = {
+    # options for the current Nixlet
+  };
+  # overwriting the default of dependency Nixlets (the user can still overwrite this using mkForce for example)
+  config."postgres".replicaCount = 10;
+```

docs/options.md

@@ -0,0 +1,3 @@
+# Options
+
+{{ include_raw("options.md") }}

lib/default.nix

@@ -2,25 +2,68 @@
   lib,
   kubenix,
   ...
-} @ attrs:
+} @ attrs: let
-with lib; rec {
+  inherit (lib) mkOption types evalModules concatMapStringsSep assertMsg;
-  evalValues = file: {rawValues, ...} @ args: (lib.evalModules {
+  nixlet-lib = rec {
-    specialArgs = {
+    nixletModule = ./nixletModule.nix;
+
+    evalValues = file: {
+      rawValues,
+      dependencies,
+      args,
+      check ? true,
+      ...
+    }: let
+      moduleArgs =
+        args
+        // {
           utils = import ./utils.nix attrs;
         };
-    modules = [
+      # get the values from the dependencies, then import them nested
-      file
+      # (so you can set postgres.replicaCount in values.nix for example when adding "postgres" as dependency)
-      (_: {
+      extraModules = map (depName: {
-        # pass through all args to the values.nix module
+        options.${depName} = mkOption {
-        config =
+          type = types.submodule {
-          rawValues
+            imports = ["${dependencies.${depName}.path}/values.nix"];
+            _module.args =
+              moduleArgs
               // {
-            _module.args = args;
+                # make sure that dependencies see their own name and version etc.
+                nixlet = {
+                  inherit (dependencies.${depName}) name version description;
+                  inherit (moduleArgs.nixlet) project;
                 };
-      })
+              };
-    ];
+          };
-  });
+          default = {};
-  mkValues = file: args: (evalValues file args).config;
+          description = let
+            n = dependencies.${depName};
+          in ''
+            Imported Nixlet as a dependency:
+
+            |Name|Version|Description|
+            |----|-------|-----------|
+            |${n.name}|${n.version}|${n.description}|
+          '';
+        };
+      }) (builtins.attrNames dependencies);
+    in
+      builtins.addErrorContext "[nixlets] while evaluating values" (
+        evalModules {
+          modules =
+            [
+              file
+              {
+                _module = {
+                  args = moduleArgs;
+                  inherit check;
+                };
+              }
+              {config = rawValues;}
+            ]
+            ++ extraModules;
+        }
+      );
 
     # wraps mkNixletInner to allow passing either a path or an attrset
     mkNixlet = arg:
@@ -50,10 +93,24 @@ with lib; rec {
         project = defaultProject;
       };
       nixlet = {
+        _type = "nixlet";
         inherit name version description path;
+        # just values of the current nixlet (lighweight)
         values = evalValues "${path}/values.nix" {
           rawValues = {};
-        nixlet = baseNixletArg;
+          dependencies = {};
+          # no checking since this doesn't include dependencies
+          check = false;
+          args.nixlet = baseNixletArg;
+        };
+        # full values, including dependencies etc. (complex)
+        fullValues = args: let
+          evaled = nixlet.eval args;
+        in
+          evalValues "${path}/values.nix" {
+            rawValues = {};
+            inherit (evaled.config.nixlet) dependencies;
+            args.nixlet = baseNixletArg;
           };
         mkDocs = opts: mkDocs (opts // {inherit nixlet;});
         eval = {
@@ -62,35 +119,47 @@ with lib; rec {
           overrides ? (_: {}),
           values ? {},
         }:
-        assert lib.assertMsg (project != null) "No default project set, please pass a project to the render method"; let
+          assert assertMsg (project != null) "No default project set, please pass a project to the eval/render method"; let
             nixletArg = baseNixletArg // {inherit project;};
           in
+            builtins.addErrorContext "[nixlets] while evaluating nixlet ${name}" (
               kubenix.evalModules.${system} {
-            module = {kubenix, ...}: {
+                module = {
+                  config,
+                  kubenix,
+                  ...
+                }: {
                   imports = with kubenix.modules; [
                     k8s
                     helm
                     docker
                     files
                     ./secretsModule.nix
-                (_: let
+                    ./nixletModule.nix
-                  finalValues = mkValues "${path}/values.nix" {
+                    (let
+                      finalValues =
+                        (evalValues "${path}/values.nix" {
                           rawValues = values;
-                    nixlet = nixletArg;
+                          inherit (config.nixlet) dependencies;
-                  };
+                          args.nixlet = nixletArg;
+                        }).config;
                     in {
                       imports = [path];
-                  _module.args.nixlet =
+                      _module.args = {
+                        nixlet =
                           {
                             values = finalValues;
                           }
                           // nixletArg;
+                        inherit nixlet-lib system;
+                      };
                     })
                     overrides
                   ];
                   kubenix.project = project;
                 };
-          };
+              }
+            );
         render = {
           system,
           project ? defaultProject,
@@ -105,6 +174,7 @@ with lib; rec {
           .resultYAML;
         # combines all secrets files in a single directory
         secrets = args: (nixlet.eval args).config.kubernetes.secretsCombined;
+
       };
     in
       nixlet;
@@ -133,8 +203,8 @@ with lib; rec {
             exit 1
           fi
         ''
-      + lib.concatStringsSep "\n" (
+        + concatMapStringsSep "\n" (
-        builtins.map (nixlet:
+          (nixlet:
             with nixlet; ''
               URL="https://gitlab.com/api/v4/projects/${projectId}/packages/generic/${name}/${version}/${name}.tar.gz"
               if ${pkgs.curl}/bin/curl --output /dev/null --silent --head --fail --header "$AUTH_HEADER" $URL; then
@@ -153,4 +223,6 @@ with lib; rec {
 
     mkDocs = opts:
       import ./valuesDocs.nix (opts // {inherit lib;});
-}
+  };
+in
+  nixlet-lib

lib/nixletModule.nix

@@ -0,0 +1,75 @@
+{
+  lib,
+  config,
+  nixlet,
+  system,
+  ...
+}: let
+  inherit (lib) mkOption types mkOptionType isType mkMerge mapAttrs mkIf literalExpression;
+  cfg = config.nixlet;
+
+  nixletType = mkOptionType {
+    name = "nixlet";
+    description = "reference";
+    descriptionClass = "noun";
+    check = isType "nixlet";
+  };
+in {
+  imports = [
+    {
+      # shortcut, allows accessing deps a bit shorter/more easily
+      _module.args.deps = cfg.deps;
+    }
+  ];
+  options.nixlet = {
+    dependencies = mkOption {
+      type = types.attrsOf nixletType;
+      default = {};
+      description = ''
+        Import other nixlets as dependencies. Works similar to Helm, specify values for these
+        Nixlets by using their name as a prefix. Like `postgres.replicaCount` in `values.nix` for example.
+      '';
+      example = literalExpression ''
+        {
+          "postgres" = nixlet-lib.mkNixlet <path>;
+          "mongodb" = nixlet-lib.fetchNixlet ...; # etc.
+        }
+      '';
+    };
+    deps = mkOption {
+      readOnly = true;
+      type = types.attrsOf types.attrs;
+      default = mapAttrs (name: val:
+        builtins.addErrorContext "[nixlets] while evaluating dependency ${name}"
+        (val.eval {
+          inherit system;
+          inherit (config.kubenix) project;
+          values = nixlet.values.${name};
+        }).config)
+      cfg.dependencies;
+      description = ''
+        Evaluated dependency nixlets. Allows accessing their resources like for example:
+
+        ```nix
+        config.nixlet.deps."<name>".kubernetes.resources
+        ```
+      '';
+    };
+    depAutoMerge = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Whether to automatically merge dependency nixlets' configs
+        with the current nixlet. If disabled, you can access dependency outputs via:
+
+        ```nix
+        config.nixlet.deps."<name>".kubernetes.resources
+        ```
+      '';
+    };
+  };
+
+  config = mkIf cfg.depAutoMerge {
+    kubernetes.resources = mkMerge (map (dep: dep.kubernetes.resources) (builtins.attrValues cfg.deps));
+  };
+}

lib/valuesDocs.nix

@@ -1,6 +1,8 @@
 {
   lib,
   nixlet,
+  # whether to generate docs for the full values, including dependencies
+  fullValues ? false,
   transformOptions ? opt: opt,
   filter ? _: true,
   headingDepth ? 3,
@@ -13,7 +15,12 @@
     mapAttrsToList
     concatStrings
     replicate
+    optionalString
+    optionAttrSetToDocList
+    attrByPath
+    generators
     ;
+  inherit (generators) toPretty;
 
   _transformOptions = opt:
     transformOptions (opt
@@ -25,7 +32,12 @@
         name = lib.removePrefix "config." opt.name;
       });
 
-  rawOpts = lib.optionAttrSetToDocList nixlet.values.options;
+  valueSource =
+    if fullValues
+    # TODO: get rid of system, just here cuz of kubenix
+    then (nixlet.fullValues {system = "x86_64-linux";})
+    else nixlet.values;
+  rawOpts = optionAttrSetToDocList valueSource.options;
   transformedOpts = map _transformOptions rawOpts;
   filteredOpts = lib.filter (opt: opt.visible && !opt.internal) transformedOpts;
 
@@ -58,7 +70,20 @@
       ${opt.type}
       ```
     ''
-    + (lib.optionalString (opt ? default && opt.default != null) ''
+    # used to show what changes a nixlet did to values of dependencies
+    + (let
+      val = toPretty {} (attrByPath opt.loc "_not found_" valueSource.config);
+      default = removeSuffix "\n" opt.default.text;
+    in
+      optionalString (opt.type != "submodule" && val != default)
+      ''
+        **Overridden value**:
+
+        ```nix
+        ${val}
+        ```
+      '')
+    + (optionalString (opt ? default && opt.default != null) ''
 
       **Default value**:
 
@@ -66,7 +91,7 @@
       ${removeSuffix "\n" opt.default.text}
       ```
     '')
-    + (lib.optionalString (opt ? example) ''
+    + (optionalString (opt ? example) ''
 
       **Example value**:
 

nix/repo/docs.nix

@@ -3,8 +3,22 @@
   cell,
   ...
 }: let
-  inherit (inputs) doclib;
+  inherit (inputs) pkgs doclib nixlet-lib;
   inherit (cell) nixlets;
+
+  optionsDoc = doclib.mkOptionDocs {
+    module = nixlet-lib.nixletModule;
+    roots = [
+      {
+        url = "https://gitlab.com/TECHNOFAB/nixlets/-/blob/main/lib";
+        path = "${inputs.self}/lib";
+      }
+    ];
+  };
+  optionsDocs = pkgs.runCommand "options-docs" {} ''
+    mkdir -p $out
+    ln -s ${optionsDoc} $out/options.md
+  '';
 in
   (doclib.mkDocs {
     docs."default" = {
@@ -23,9 +37,13 @@ in
           domains = ["nixlets.projects.tf"];
         };
       };
+      macros = {
+        enable = true;
+        includeDir = toString optionsDocs;
+      };
       dynamic-nav = {
         enable = true;
-        files."Nixlets Values" = builtins.map (val: {${val.name} = val.mkDocs {};}) (builtins.attrValues nixlets);
+        files."Nixlets Values" = builtins.map (val: {${val.name} = val.mkDocs {fullValues = true;};}) (builtins.attrValues nixlets);
       };
       config = {
         site_name = "Nixlets";
@@ -43,8 +61,10 @@ in
           {"Creating Nixlets" = "creation.md";}
           {"Packaging" = "packaging.md";}
           {"Usage" = "usage.md";}
+          {"Importing" = "importing.md";}
           {"Generating Docs" = "generating_docs.md";}
           {"Secrets" = "secrets.md";}
+          {"Options" = "options.md";}
         ];
         markdown_extensions = [
           {

tests/fixtures/dependency/default.nix

@@ -0,0 +1,3 @@
+{nixlet-lib, ...}: {
+  config.nixlet.dependencies."example" = nixlet-lib.mkNixlet ../example;
+}

tests/fixtures/dependency/nixlet.nix

@@ -0,0 +1,6 @@
+{
+  name = "dep";
+  version = "0.0.1";
+  description = "hello world";
+  defaultProject = "dep";
+}

tests/fixtures/dependency/values.nix

@@ -0,0 +1,5 @@
+_: {
+  options = {};
+
+  config."example".example = "Hello dependency!";
+}

tests/fixtures/example/default.nix

@@ -0,0 +1,3 @@
+{nixlet, ...}: with nixlet; {
+  kubernetes.resources.configMaps."test".data."test" = values.example;
+}

tests/lib_test.nix

@@ -3,11 +3,14 @@
   ntlib,
   nixlet-lib,
   ...
-}: {
+}: let
+  inherit (pkgs.lib) mkForce;
+in {
   suites."Lib Tests" = {
     pos = __curPos;
     tests = let
       nixlet = nixlet-lib.mkNixlet ./fixtures/example;
+      depNixlet = nixlet-lib.mkNixlet ./fixtures/dependency;
     in [
       {
         name = "mkNixlet fail on nonexistant nixlet.nix";
@@ -42,6 +45,27 @@
             assert_file_contains "${docs}" '"Hello world!"'
           '';
       }
+      {
+        name = "Nixlet dependencies";
+        expected = "Hello dependency!";
+        actual = let
+          evaled = depNixlet.eval {inherit (pkgs.stdenv.hostPlatform) system;};
+        in
+          evaled.config.kubernetes.resources.configMaps."test".data."test";
+      }
+      {
+        name = "Nixlet dependency value override";
+        expected = "Hello override!";
+        actual = let
+          evaled = depNixlet.eval {
+            inherit (pkgs.stdenv.hostPlatform) system;
+            values = {
+              "example".example = mkForce "Hello override!";
+            };
+          };
+        in
+          evaled.config.kubernetes.resources.configMaps."test".data."test";
+      }
     ];
   };
 }