Merge branch 'feat/autodiscovery' into 'main'

feat: add autodiscovery block

See merge request rensa-nix/core!3

cells/repo/docs.nix

@@ -36,6 +36,7 @@ in
           {"API Reference" = "api.md";}
           {"Related Libraries" = "libraries.md";}
           {"Direnv Integration" = "direnv.md";}
+          {"Debugging" = "debugging.md";}
         ];
         markdown_extensions = [
           {

docs/api.md

@@ -8,7 +8,7 @@ The main entry point for creating a Rensa flake.
 
 - `inputs`: The flake inputs.
 - `cellsFrom`: Path to the directory containing your cells.
-- `cellBlocks`: A list of blocks to load for each cell.
+- `cellBlocks`: A list of blocks to load for each cell. Defaults to `[rensa.blocks.autodiscover]`.
 - `transformInputs` (optional): A function to transform inputs before they are passed to cells.
 
 ### Returns
@@ -23,7 +23,7 @@ The underlying builder function used by `buildWith`. It returns the raw Rensa ou
 
 - `inputs`: The flake inputs.
 - `cellsFrom`: Path to the directory containing your cells.
-- `cellBlocks`: A list of blocks to load for each cell.
+- `cellBlocks`: A list of blocks to load for each cell. Defaults to `[rensa.blocks.autodiscover]`.
 - `transformInputs` (optional): A function to transform inputs.
 
 ### Returns
@@ -56,6 +56,45 @@ by passing through actions etc. from a cell.
 
 Currently not really used.
 
+### `autodiscover`
+
+```nix
+autodiscover
+# or
+(autodiscover "cellName")
+```
+
+Automatically discovers and loads all cell blocks by scanning the cells directory for `.nix` files.
+When `autodiscover` is present in `cellBlocks`, Rensa will:
+
+1. Walk through all cells in `cellsFrom` (or specific cell if provided).
+1. Find all `.nix` files (excluding `flake.nix`).
+1. Find all directories containing `default.nix`.
+1. Generate `simple` block definitions for each discovered block.
+
+**Usage patterns:**
+
+```nix
+# full autodiscovery
+cellBlocks = with rensa.blocks; [
+  autodiscover
+];
+# or
+cellBlocks = with rensa.blocks; [
+  (autodiscover "backend")  # only discover blocks from backend cell
+  (autodiscover "frontend") # only discover blocks from frontend cell
+];
+# mixed
+cellBlocks = with rensa.blocks; [
+  (simple "myBlock")  # explicit block (takes precedence)
+  autodiscover        # discover all others
+];
+```
+
+!!! note
+
+    When a block is both explicitly defined and discovered, the explicit definition takes precedence.
+
 ## `rensa.select`
 
 Helper to select specific outputs from the generated flake.

docs/debugging.md

@@ -0,0 +1,38 @@
+# Debugging
+
+## Error Context
+
+To help with debugging, Rensa uses Nix's `builtins.addErrorContext` to add context
+about where an error is happening.
+
+Use the following command to get just these logs from Rensa and please include that (or the whole stacktrace) in any issues or bug reports :)
+
+```sh
+nix ... --show-trace 2> >(grep "… \[ren\]")
+```
+
+## Trace Verbose
+
+There are also some `traceVerbose` calls in Rensa, so to debug non-failing executions
+just run Nix with the `--trace-verbose` flag.
+
+The output could look like this (example with this repo's flake):
+
+```sh
+trace: [ren] loading cell block ci, type ci, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-darwin
+trace: [ren] loading cell block devShells, type devShells, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-darwin
+trace: [ren] loading cell block docs, type docs, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-darwin
+trace: [ren] loading cell block soonix, type soonix, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-darwin
+trace: [ren] loading cell block ci, type ci, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-linux
+trace: [ren] loading cell block devShells, type devShells, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-linux
+trace: [ren] loading cell block docs, type docs, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-linux
+trace: [ren] loading cell block soonix, type soonix, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system aarch64-linux
+trace: [ren] loading cell block ci, type ci, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-darwin
+trace: [ren] loading cell block devShells, type devShells, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-darwin
+trace: [ren] loading cell block docs, type docs, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-darwin
+trace: [ren] loading cell block soonix, type soonix, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-darwin
+trace: [ren] loading cell block ci, type ci, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-linux
+trace: [ren] loading cell block devShells, type devShells, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-linux
+trace: [ren] loading cell block docs, type docs, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-linux
+trace: [ren] loading cell block soonix, type soonix, from cell /nix/store/7acz6q9360fkg0x187yx43d1vwpv850l-cells/repo, for system x86_64-linux
+```

lib/blocks/default.nix

@@ -6,6 +6,17 @@
   dynamic = name: {
     inherit name;
     type = name;
+    cli = true;
+    actions = args: {};
     # TODO: dynamic actions
   };
+  autodiscover = {
+    name = "__autodiscover";
+    type = "__autodiscover";
+    _functor = self: cell:
+      self
+      // {
+        inherit cell;
+      };
+  };
 }

lib/core/autodiscover.nix

@@ -0,0 +1,98 @@
+{
+  l,
+  cellsFrom,
+  cellBlocks,
+}: let
+  # find autodiscover blocks
+  autodiscoverBlocks = l.filter (block: block.name or "" == "__autodiscover") cellBlocks;
+  explicitBlocks = l.filter (block: block.name or "" != "__autodiscover") cellBlocks;
+  hasAutodiscover = (l.length autodiscoverBlocks) > 0;
+
+  # discover all block names from a single cell's directory
+  discoverBlocksFromCell = cellPath: let
+    cellContents = l.readDir cellPath;
+    # filter for .nix files (but not flake.nix) and dirs with default.nix
+    blockNames = l.unique (
+      l.filter (name: name != null) (
+        l.mapAttrsToList (
+          name: type:
+            if type == "regular" && l.hasSuffix ".nix" name && name != "flake.nix"
+            then l.removeSuffix ".nix" name
+            else if type == "directory" && l.pathExists (cellPath + "/${name}/default.nix")
+            then name
+            else null
+        )
+        cellContents
+      )
+    );
+  in
+    builtins.addErrorContext "[ren] while discovering blocks from cell ${cellPath}"
+    blockNames;
+
+  # discover all blocks from all cells or specific cells
+  discoverAllBlocks = let
+    # check if any autodiscover blocks have cell-specific targets (eg. `autodiscover "a"`)
+    cellSpecificDiscoveries = l.filter (block: block ? cell) autodiscoverBlocks;
+    globalDiscoveries = l.filter (block: !(block ? cell)) autodiscoverBlocks;
+
+    cellSpecificBlocks = l.flatten (
+      l.map (
+        block: let
+          cellPath = cellsFrom + "/${block.cell}";
+        in
+          if l.pathExists cellPath
+          then discoverBlocksFromCell cellPath
+          else []
+      )
+      cellSpecificDiscoveries
+    );
+
+    # global autodiscover
+    globalBlocks = builtins.addErrorContext "[ren] while discovering global blocks" (
+      if (l.length globalDiscoveries) > 0
+      then let
+        cells = l.readDir cellsFrom;
+        allBlockNames = l.unique (
+          l.flatten (
+            l.mapAttrsToList (
+              cellName: cellType:
+                if cellType == "directory"
+                then discoverBlocksFromCell (cellsFrom + "/${cellName}")
+                else []
+            )
+            cells
+          )
+        );
+      in
+        allBlockNames
+      else []
+    );
+
+    allBlockNames = l.unique (cellSpecificBlocks ++ globalBlocks);
+  in
+    builtins.addErrorContext "[ren] while discovering all blocks"
+    l.map (name: {
+      inherit name;
+      type = name;
+    })
+    allBlockNames;
+
+  discoveredBlocks =
+    if hasAutodiscover
+    then discoverAllBlocks
+    else [];
+
+  # merge the explicit and autodiscovered blocks (explicit taking precedence)
+  allBlocks = explicitBlocks ++ discoveredBlocks;
+  uniqueBlocks =
+    l.foldl' (
+      acc: block:
+      # first occurence of name wins
+        if l.any (b: b.name == block.name) acc
+        then acc
+        else acc ++ [block]
+    ) []
+    allBlocks;
+in
+  builtins.addErrorContext "[ren] while autodiscovering cell blocks"
+  uniqueBlocks

lib/core/builder.nix

@@ -2,6 +2,8 @@
   l,
   utils,
   loader,
+  autodiscover,
+  blocks,
 }: let
   inherit (utils) accumulate;
   inherit (loader) createCellLoader;
@@ -9,7 +11,7 @@
   build = {
     inputs,
     cellsFrom,
-    cellBlocks,
+    cellBlocks ? [blocks.autodiscover],
     transformInputs ? system: i: i,
     ...
   } @ args: let
@@ -28,26 +30,31 @@
         ]
       );
 
+    cellBlocks' = autodiscover {inherit l cellsFrom cellBlocks;};
+
     cells = res.output;
 
     loadOutputFor = system: let
       loadCell = createCellLoader {
-        inherit inputs system cells cellsFrom cellBlocks transformInputs;
+        inherit inputs system cells cellsFrom transformInputs;
+        cellBlocks = cellBlocks';
       };
 
       cells' = l.mapAttrsToList (cell: type: cell) (l.readDir cellsFrom);
       res = accumulate (l.map loadCell cells');
-    in [
-      {${system} = res.output;}
-      {${system} = res.actions;}
-      {
-        name = system;
-        value = res.init;
-      }
-    ];
+    in
+      builtins.addErrorContext "[ren] while loading output for ${system}" [
+        {${system} = res.output;}
+        {${system} = res.actions;}
+        {
+          name = system;
+          value = res.init;
+        }
+      ];
 
     res = accumulate (l.map loadOutputFor systems);
   in
+    builtins.addErrorContext "[ren] while building rensa output"
     res.output
     // {
       __ren = {
@@ -89,10 +96,12 @@
     recursiveUpdate = lhs: rhs:
       recursiveUpdateUntil g lhs rhs;
   in
-    build args
-    // {
-      __functor = l.flip recursiveUpdate;
-    };
+    builtins.addErrorContext "[ren] while building rensa output (buildWith)" (
+      build args
+      // {
+        __functor = l.flip recursiveUpdate;
+      }
+    );
 in {
   inherit build buildWith;
 }

lib/core/call-flake.nix

@@ -112,4 +112,5 @@ ref: inputOverrides: let
     )
     lockFile.nodes;
 in
+  builtins.addErrorContext "[ren] while loading flake lockfile ${ref}"
   allNodes.${lockFile.root}

lib/core/default.nix

@@ -1,11 +1,13 @@
 {
   l,
   utils,
+  blocks,
 }: let
   paths = import ./paths.nix;
   callFlake = import ./call-flake.nix;
+  autodiscover = import ./autodiscover.nix;
   loader = import ./loader.nix {inherit l utils paths callFlake;};
-  builder = import ./builder.nix {inherit l utils loader;};
+  builder = import ./builder.nix {inherit l utils loader autodiscover blocks;};
 in {
   inherit (builder) build buildWith;
 }

lib/core/loader.nix

@@ -33,14 +33,18 @@
             }).outputs
           else {};
       in
-        importSignatureFor system updatedCell cells additionalInputs;
+        builtins.addErrorContext "[ren] while getting input signature for ${blockP}" (
+          importSignatureFor system updatedCell cells additionalInputs
+        );
 
       import' = importPath: let
         block = import importPath;
       in
         if l.typeOf block == "set"
         then block
-        else block signature;
+        else
+          builtins.addErrorContext "[ren] while importing block at ${importPath}"
+          (block signature);
 
       importPaths =
         if isFile
@@ -101,18 +105,20 @@
     in
       optionalLoad (isFile || isDir) (
         assert l.assertMsg isAttrs "cell block does not return an attrset: ${importPaths.displayPath}";
-          l.traceVerbose "[ren] loading cell block ${cellBlock.name}, type ${cellBlock.type}, from cell ${cellP}"
-          [
-            {${cellBlock.name} = imported;}
-            {${cellBlock.name} = l.mapAttrs (_: set: set.actions) extracted;}
-            ({
-                cellBlock = cellBlock.name;
-                blockType = cellBlock.type;
-                targets = l.mapAttrsToList (_: set: set.init) extracted;
-              }
-              // (l.optionalAttrs (l.pathExists blockP.readmeDir) {readme = blockP.readmeDir;})
-              // (l.optionalAttrs (l.pathExists blockP.readme) {inherit (blockP) readme;}))
-          ]
+          l.traceVerbose "[ren] loading cell block ${cellBlock.name}, type ${cellBlock.type}, from cell ${cellP}, for system ${system}" (
+            builtins.addErrorContext "[ren] loading cell block ${cellBlock.name}, type ${cellBlock.type}, from cell ${cellP}, for system ${system}"
+            [
+              {${cellBlock.name} = imported;}
+              {${cellBlock.name} = l.mapAttrs (_: set: set.actions) extracted;}
+              ({
+                  cellBlock = cellBlock.name;
+                  blockType = cellBlock.type;
+                  targets = l.mapAttrsToList (_: set: set.init) extracted;
+                }
+                // (l.optionalAttrs (l.pathExists blockP.readmeDir) {readme = blockP.readmeDir;})
+                // (l.optionalAttrs (l.pathExists blockP.readme) {inherit (blockP) readme;}))
+            ]
+          )
       );
   in
     loadCellBlock;
@@ -133,7 +139,7 @@
       # fixes `ìnfinite recursion` errors when accessing cell attributes from sibling blocks
       # example: cells/test/a.nix returns `cell.b`, so the same thing as cells/test/b.nix.
       # this would previously fail with infinite recursion, this makes it work:
-      cell = l.listToAttrs (l.concatMap (
+      cell = builtins.addErrorContext "[ren] while accessing cell '${cellName}' siblings" (l.listToAttrs (l.concatMap (
           block: let
             blockP = paths.cellBlockPath cellP block;
             exists = l.pathExists blockP.file || l.pathExists blockP.dir;
@@ -148,18 +154,19 @@
             ]
             else []
         )
-        cellBlocks');
+        cellBlocks'));
       loadCellBlock = createCellBlockLoader {inherit inputs system cells cell transformInputs;};
       res = accumulate (l.map (loadCellBlock cellName cellP) cellBlocks');
-    in [
-      {${cellName} = res.output;}
-      {${cellName} = res.actions;}
-      ({
-          cell = cellName;
-          cellBlocks = res.init;
-        }
-        // (l.optionalAttrs (l.pathExists cellP.readme) {inherit (cellP) readme;}))
-    ];
+    in
+      builtins.addErrorContext "[ren] while loading cell ${cellName}" [
+        {${cellName} = res.output;}
+        {${cellName} = res.actions;}
+        ({
+            cell = cellName;
+            cellBlocks = res.init;
+          }
+          // (l.optionalAttrs (l.pathExists cellP.readme) {inherit (cellP) readme;}))
+      ];
   in
     loadCell;
 in {

lib/default.nix

@@ -1,9 +1,9 @@
 {lib}: let
   l = builtins // lib;
-  utils = import ./utils {inherit l;};
-  core = import ./core {inherit l utils;};
-  compat = import ./compat {inherit l;};
   blocks = import ./blocks;
+  utils = import ./utils {inherit l;};
+  compat = import ./compat {inherit l;};
+  core = import ./core {inherit l utils blocks;};
 in {
   inherit (compat) filter select get;
   inherit (core) build buildWith;

tests/autodiscovery_test.nix

@@ -0,0 +1,63 @@
+{
+  pkgs,
+  ntlib,
+  ...
+}: {
+  suites."Autodiscovery" = {
+    pos = __curPos;
+    tests = [
+      {
+        name = "find nix files";
+        type = "script";
+        script = ''
+          ${ntlib.helpers.path [pkgs.gnugrep pkgs.coreutils pkgs.nix]}
+          ${ntlib.helpers.scriptHelpers}
+
+          mkdir -p "testcell"
+          echo '{ foo = "bar"; }' > "testcell/packages.nix"
+          echo '{ baz = "qux"; }' > "testcell/devShells.nix"
+          echo '{ ignored = true; }' > "testcell/flake.nix"
+
+          result=$(nix eval --impure --expr '
+            let
+              lib = import ${pkgs.path}/lib;
+              autodiscover = import ${../lib/core/autodiscover.nix} {
+                l = builtins // lib;
+                cellsFrom = ./.;
+                cellBlocks = [ { name = "__autodiscover"; } ];
+              };
+            in
+              builtins.length (builtins.filter (b: b.name == "packages" || b.name == "devShells") autodiscover)
+          ')
+
+          assert_eq "$result" "2" "should discover 2 blocks"
+        '';
+      }
+      {
+        name = "finds directories";
+        type = "script";
+        script = ''
+          ${ntlib.helpers.path [pkgs.gnugrep pkgs.coreutils pkgs.nix]}
+          ${ntlib.helpers.scriptHelpers}
+
+          mkdir -p "testcell/packages"
+          echo '{ hello = "world"; }' > "testcell/packages/default.nix"
+
+          result=$(nix eval --impure --expr '
+            let
+              lib = import ${pkgs.path}/lib;
+              autodiscover = import ${../lib/core/autodiscover.nix} {
+                l = builtins // lib;
+                cellsFrom = ./.;
+                cellBlocks = [ { name = "__autodiscover"; } ];
+              };
+            in
+              builtins.any (b: b.name == "packages") autodiscover
+          ')
+
+          assert "$result == true" "Should discover directory with default.nix"
+        '';
+      }
+    ];
+  };
+}