From 98141a1f5c0c1b48837cbc363cbb8ebf74b5d044 Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Thu, 12 Jun 2025 21:23:28 +0200
Subject: [PATCH 1/8] feat: switch to module system to evaluate suites & tests

add autodiscovery feature
---
 flake.nix           |  25 +++++-
 lib/default.nix     |  94 +++++++++++-----------
 lib/flakeModule.nix |  52 ++----------
 lib/lib_test.nix    |  44 ++++++++++
 lib/module.nix      | 192 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 312 insertions(+), 95 deletions(-)
 create mode 100644 lib/lib_test.nix
 create mode 100644 lib/module.nix

diff --git a/flake.nix b/flake.nix
index bcc7b54..e12b9ae 100644
--- a/flake.nix
+++ b/flake.nix
@@ -236,7 +236,7 @@
         ci = {
           stages = ["test" "build" "deploy"];
           jobs = {
-            "test" = {
+            "test:flakeModule" = {
               stage = "test";
               script = [
                 "nix run .#nixtests:run -- --junit=junit.xml"
@@ -247,6 +247,17 @@
                 reports.junit = "junit.xml";
               };
             };
+            "test:lib" = {
+              stage = "test";
+              script = [
+                "nix run .#lib-tests -- --junit=junit.xml"
+              ];
+              allow_failure = true;
+              artifacts = {
+                when = "always";
+                reports.junit = "junit.xml";
+              };
+            };
             "test:go" = {
               stage = "test";
               nix.deps = with pkgs; [go go-junit-report gocover-cobertura];
@@ -300,7 +311,17 @@
           };
         };
 
-        packages.default = pkgs.callPackage ./package.nix {};
+        packages = let
+          ntlib = import ./lib {inherit pkgs lib;};
+        in {
+          default = pkgs.callPackage ./package.nix {};
+          lib-tests = ntlib.mkNixtest {
+            modules = ntlib.autodiscover {dir = ./lib;};
+            args = {
+              inherit pkgs;
+            };
+          };
+        };
       };
     };
 
diff --git a/lib/default.nix b/lib/default.nix
index e8b7f97..011d50e 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -1,62 +1,64 @@
 {
   pkgs,
   lib ? pkgs.lib,
-  self ? "",
   ...
-}: {
-  mkTest = {
-    type ? "unit",
-    name,
-    description ? "",
-    format ? "json",
-    expected ? null,
-    actual ? null,
-    actualDrv ? null,
-    script ? null,
-    pos ? null,
+}: let
+  inherit (lib) evalModules toList;
+in rec {
+  mkBinary = {
+    nixtests,
+    extraParams,
   }: let
-    fileRelative = lib.removePrefix ((toString self) + "/") pos.file;
-    actual' =
-      if format == "json"
-      then actual
-      else lib.generators.toPretty {} actual;
-    expected' =
-      if format == "json"
-      then expected
-      else lib.generators.toPretty {} expected;
+    program = pkgs.callPackage ../package.nix {};
   in
-    assert lib.assertMsg (!(type == "script" && script == null)) "test ${name} has type 'script' but no script was passed"; {
-      inherit type name description;
-      actual = actual';
-      expected = expected';
-      # discard string context, otherwise it's being built instantly which we don't want
-      actualDrv = builtins.unsafeDiscardStringContext (actualDrv.drvPath or "");
-      script =
-        if script != null
-        then
-          builtins.unsafeDiscardStringContext
-          (pkgs.writeShellScript "nixtest-${name}" ''
-            # show which line failed the test
-            set -x
-            ${script}
-          '').drvPath
-        else null;
-      pos =
-        if pos == null
-        then ""
-        else "${fileRelative}:${toString pos.line}";
+    (pkgs.writeShellScriptBin "nixtests:run" ''
+      ${program}/bin/nixtest --tests=${nixtests} ${extraParams} "$@"
+    '')
+    // {
+      tests = nixtests;
     };
-  mkSuite = name: tests: {
-    inherit name tests;
-  };
+
   exportSuites = suites: let
     suitesList =
       if builtins.isList suites
       then suites
       else [suites];
-    testsMapped = builtins.toJSON suitesList;
+    suitesMapped = builtins.toJSON suitesList;
   in
     pkgs.runCommand "tests.json" {} ''
-      echo '${testsMapped}' > $out
+      echo '${suitesMapped}' > $out
     '';
+
+  module = import ./module.nix {inherit lib pkgs;};
+
+  autodiscover = {
+    dir,
+    pattern ? ".*_test.nix",
+  }: let
+    files = builtins.readDir dir;
+    matchingFiles = builtins.filter (name: builtins.match pattern name != null) (builtins.attrNames files);
+    imports = map (file: /${dir}/${file}) matchingFiles;
+  in {
+    inherit imports;
+    # automatically set the base so test filepaths are easier to read
+    config.base = builtins.toString dir + "/";
+  };
+
+  mkNixtestConfig = {
+    modules,
+    args,
+    ...
+  }:
+    (evalModules {
+      modules =
+        (toList modules)
+        ++ [
+          module
+          {
+            _module.args = args;
+          }
+        ];
+    }).config;
+
+  mkNixtest = args: (mkNixtestConfig args).app;
 }
diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix
index 6068aea..1d7e406 100644
--- a/lib/flakeModule.nix
+++ b/lib/flakeModule.nix
@@ -15,56 +15,14 @@ in {
       nixtests-lib = import ./. {inherit pkgs self;};
     in {
       options.nixtest = mkOption {
-        type = types.submodule ({...}: {
-          options = {
-            skip = mkOption {
-              type = types.str;
-              default = "";
-              description = "Which tests to skip (regex)";
-            };
-            suites = mkOption {
-              type = types.attrsOf (types.submodule {
-                options = {
-                  tests = mkOption {
-                    type = types.listOf types.attrs;
-                    default = [];
-                  };
-                  pos = mkOption {
-                    type = types.nullOr types.attrs;
-                    default = null;
-                  };
-                };
-              });
-              default = {};
-            };
-          };
-        });
+        type = types.submodule (nixtests-lib.module);
         default = {};
       };
+      config.nixtest.base = toString self + "/";
 
-      config.legacyPackages = rec {
-        "nixtests" = let
-          suites = map (suiteName: let
-            suite = builtins.getAttr suiteName config.nixtest.suites;
-          in
-            nixtests-lib.mkSuite
-            suiteName
-            (map (test:
-              nixtests-lib.mkTest ({
-                  # default pos to suite's pos if given
-                  pos = suite.pos;
-                }
-                // test))
-            suite.tests))
-          (builtins.attrNames config.nixtest.suites);
-        in
-          nixtests-lib.exportSuites suites;
-        "nixtests:run" = let
-          program = pkgs.callPackage ./../package.nix {};
-        in
-          pkgs.writeShellScriptBin "nixtests:run" ''
-            ${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
-          '';
+      config.legacyPackages = {
+        "nixtests" = config.nixtest.finalConfigJson;
+        "nixtests:run" = config.nixtest.app;
       };
     }
   );
diff --git a/lib/lib_test.nix b/lib/lib_test.nix
new file mode 100644
index 0000000..6a47538
--- /dev/null
+++ b/lib/lib_test.nix
@@ -0,0 +1,44 @@
+{
+  pkgs,
+  lib,
+  ...
+}: let
+  ntlib = import ./. {inherit pkgs lib;};
+in {
+  suites."Lib Tests".tests = [
+    {
+      name = "autodiscovery";
+      type = "script";
+      script = let
+        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
+          dir = ./.;
+        }));
+      in
+        # sh
+        ''
+          export PATH="${pkgs.gnugrep}/bin"
+          grep -q lib_test.nix ${actual}
+          grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual}
+        '';
+    }
+    {
+      name = "binary";
+      type = "script";
+      script = let
+        binary =
+          (ntlib.mkBinary {
+            nixtests = "stub";
+            extraParams = "--pure";
+          })
+          + "/bin/nixtests:run";
+      in
+        # sh
+        ''
+          export PATH="${pkgs.gnugrep}/bin"
+          grep -q nixtest ${binary}
+          grep -q -- "--pure" ${binary}
+          grep -q -- "--tests=stub" ${binary}
+        '';
+    }
+  ];
+}
diff --git a/lib/module.nix b/lib/module.nix
new file mode 100644
index 0000000..db5d5b9
--- /dev/null
+++ b/lib/module.nix
@@ -0,0 +1,192 @@
+{
+  pkgs,
+  lib,
+  ...
+}: let
+  inherit (lib) mkOptionType mkOption types;
+
+  nixtest-lib = import ./default.nix {inherit pkgs lib;};
+
+  unsetType = mkOptionType {
+    name = "unset";
+    description = "unset";
+    descriptionClass = "noun";
+    check = value: true;
+  };
+  unset = {
+    _type = "unset";
+  };
+  isUnset = lib.isType "unset";
+
+  filterUnset = value:
+    if builtins.isAttrs value && !builtins.hasAttr "_type" value
+    then let
+      filteredAttrs = builtins.mapAttrs (n: v: filterUnset v) value;
+    in
+      lib.filterAttrs (name: value: (!isUnset value)) filteredAttrs
+    else if builtins.isList value
+    then builtins.filter (elem: !isUnset elem) (map filterUnset value)
+    else value;
+
+  testsSubmodule = {
+    config,
+    testsBase,
+    pos,
+    ...
+  }: {
+    options = {
+      pos = mkOption {
+        type = types.either types.attrs unsetType;
+        default = pos;
+        apply = val:
+          if isUnset val
+          then val
+          else let
+            fileRelative = lib.removePrefix testsBase val.file;
+          in "${fileRelative}:${toString val.line}";
+      };
+      type = mkOption {
+        type = types.enum ["unit" "snapshot" "script"];
+        default = "unit";
+        apply = value:
+          assert lib.assertMsg (value != "script" || !isUnset config.script)
+          "test '${config.name}' as type 'script' requires 'script' to be set";
+          assert lib.assertMsg (value != "unit" || !isUnset config.expected)
+          "test '${config.name}' as type 'unit' requires 'expected' to be set";
+          assert lib.assertMsg (
+            let
+              actualIsUnset = isUnset config.actual;
+              actualDrvIsUnset = isUnset config.actualDrv;
+            in
+              (value != "unit")
+              || (!actualIsUnset && actualDrvIsUnset)
+              || (actualIsUnset && !actualDrvIsUnset)
+          )
+          "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value;
+      };
+      name = mkOption {
+        type = types.str;
+      };
+      description = mkOption {
+        type = types.either types.str unsetType;
+        default = unset;
+      };
+      format = mkOption {
+        type = types.enum ["json" "pretty"];
+        default = "json";
+      };
+      expected = mkOption {
+        type = types.anything;
+        default = unset;
+        apply = val:
+          if isUnset val || config.format == "json"
+          then val
+          else lib.generators.toPretty {} val;
+      };
+      actual = mkOption {
+        type = types.anything;
+        default = unset;
+        apply = val:
+          if isUnset val || config.format == "json"
+          then val
+          else lib.generators.toPretty {} val;
+      };
+      actualDrv = mkOption {
+        type = types.either types.package unsetType;
+        default = unset;
+        apply = val:
+        # keep unset value
+          if isUnset val
+          then val
+          else builtins.unsafeDiscardStringContext (val.drvPath or "");
+      };
+      script = mkOption {
+        type = types.either types.str unsetType;
+        default = unset;
+        apply = val:
+          if isUnset val
+          then val
+          else
+            builtins.unsafeDiscardStringContext
+            (pkgs.writeShellScript "nixtest-${config.name}" ''
+              # show which line failed the test
+              set -x
+              ${val}
+            '').drvPath;
+      };
+    };
+  };
+
+  suitesSubmodule = {
+    name,
+    config,
+    testsBase,
+    ...
+  }: {
+    options = {
+      name = mkOption {
+        type = types.str;
+        default = name;
+      };
+      pos = mkOption {
+        type = types.either types.attrs unsetType;
+        default = unset;
+      };
+      tests = mkOption {
+        type = types.listOf (types.submoduleWith {
+          modules = [testsSubmodule];
+          specialArgs = {
+            inherit (config) pos;
+            inherit testsBase;
+          };
+        });
+        default = [];
+      };
+    };
+  };
+
+  nixtestSubmodule = {config, ...}: {
+    options = {
+      base = mkOption {
+        description = "Base directory of the tests, will be removed from the test file path";
+        type = types.str;
+        default = "";
+      };
+      skip = mkOption {
+        type = types.str;
+        default = "";
+      };
+      suites = mkOption {
+        type = types.attrsOf (types.submoduleWith {
+          modules = [suitesSubmodule];
+          specialArgs = {
+            testsBase = config.base;
+          };
+        });
+        default = {};
+        apply = suites:
+          map (
+            n: filterUnset (builtins.removeAttrs suites.${n} ["pos"])
+          )
+          (builtins.attrNames suites);
+      };
+
+      finalConfigJson = mkOption {
+        internal = true;
+        type = types.package;
+      };
+      app = mkOption {
+        internal = true;
+        type = types.package;
+      };
+    };
+    config = {
+      finalConfigJson = nixtest-lib.exportSuites config.suites;
+      app = nixtest-lib.mkBinary {
+        nixtests = config.finalConfigJson;
+        extraParams = ''--skip="${config.skip}"'';
+      };
+    };
+  };
+in
+  nixtestSubmodule

From 001b575f31f45409b953371a56509ed3f2035f08 Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Fri, 13 Jun 2025 15:43:21 +0200
Subject: [PATCH 2/8] chore(module): remove "set -x" addition in script tests

---
 lib/module.nix | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/module.nix b/lib/module.nix
index db5d5b9..14746e4 100644
--- a/lib/module.nix
+++ b/lib/module.nix
@@ -108,11 +108,7 @@
           then val
           else
             builtins.unsafeDiscardStringContext
-            (pkgs.writeShellScript "nixtest-${config.name}" ''
-              # show which line failed the test
-              set -x
-              ${val}
-            '').drvPath;
+            (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath;
       };
     };
   };

From bed029f4a90b886bb85aef76264e5af40d0d3938 Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Fri, 13 Jun 2025 15:44:16 +0200
Subject: [PATCH 3/8] feat: add test helpers

default args to {}
---
 lib/default.nix      |  4 +++-
 lib/scriptHelpers.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++
 lib/testHelpers.nix  |  5 +++++
 3 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 lib/scriptHelpers.sh
 create mode 100644 lib/testHelpers.nix

diff --git a/lib/default.nix b/lib/default.nix
index 011d50e..2aa3683 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -5,6 +5,8 @@
 }: let
   inherit (lib) evalModules toList;
 in rec {
+  helpers = import ./testHelpers.nix {inherit lib;};
+
   mkBinary = {
     nixtests,
     extraParams,
@@ -46,7 +48,7 @@ in rec {
 
   mkNixtestConfig = {
     modules,
-    args,
+    args ? {},
     ...
   }:
     (evalModules {
diff --git a/lib/scriptHelpers.sh b/lib/scriptHelpers.sh
new file mode 100644
index 0000000..0b59829
--- /dev/null
+++ b/lib/scriptHelpers.sh
@@ -0,0 +1,51 @@
+output=
+exit_code=
+
+function assert() {
+  test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; }
+}
+function assert_eq() {
+  assert "$1 -eq $2" "$3"
+}
+function assert_not_eq() {
+  assert "$1 -ne $2" "$3"
+}
+function assert_contains() {
+  echo "$1" | grep -q -- "$2" || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_not_contains() {
+  echo "$1" | grep -q -- "$2" && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_contains() {
+  grep -q -- "$2" $1 || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_not_contains() {
+  grep -q -- "$2" $1 && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+
+function tmpdir() {
+  dir=$(mktemp -d)
+  trap "rm -rf $dir" EXIT
+  echo -n "$dir"
+}
+function tmpfile() {
+  file=$(mktemp)
+  trap "rm -f $file" EXIT
+  echo -n "$file"
+}
+function run() {
+  output=$($@ 2>&1)
+  exit_code=$?
+}
diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix
new file mode 100644
index 0000000..c29618d
--- /dev/null
+++ b/lib/testHelpers.nix
@@ -0,0 +1,5 @@
+{lib, ...}: {
+  path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
+  pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
+  scriptHelpers = builtins.readFile ./scriptHelpers.sh;
+}

From 006537e6abbcb6d6b78dd1c76e1c937b01378abd Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Fri, 13 Jun 2025 15:46:52 +0200
Subject: [PATCH 4/8] test: nest sample tests as fixtures in real tests

also use new test helpers
---
 flake.nix                      | 101 +--------------------------------
 lib/lib_test.nix               |  44 --------------
 tests/fixtures/sample_test.nix |  97 +++++++++++++++++++++++++++++++
 tests/lib_test.nix             |  86 ++++++++++++++++++++++++++++
 4 files changed, 186 insertions(+), 142 deletions(-)
 delete mode 100644 lib/lib_test.nix
 create mode 100644 tests/fixtures/sample_test.nix
 create mode 100644 tests/lib_test.nix

diff --git a/flake.nix b/flake.nix
index e12b9ae..da50a26 100644
--- a/flake.nix
+++ b/flake.nix
@@ -11,7 +11,6 @@
         inputs.nix-gitlab-ci.flakeModule
         inputs.nix-devtools.flakeModule
         inputs.nix-mkdocs.flakeModule
-        ./lib/flakeModule.nix
       ];
       systems = import systems;
       flake = {};
@@ -63,100 +62,6 @@
           };
         };
 
-        nixtest = {
-          skip = "skip.*d";
-          suites = {
-            "suite-one" = {
-              pos = __curPos;
-              tests = [
-                {
-                  name = "test-one";
-                  # required to figure out file and line, but optional
-                  expected = 1;
-                  actual = 1;
-                }
-                {
-                  name = "fail";
-                  expected = 0;
-                  actual = "meow";
-                }
-                {
-                  name = "snapshot-test";
-                  type = "snapshot";
-                  actual = "test";
-                }
-                {
-                  name = "test-snapshot-drv";
-                  type = "snapshot";
-                  actualDrv = pkgs.runCommand "test-snapshot" {} ''
-                    echo '"snapshot drv"' > $out
-                  '';
-                }
-                {
-                  name = "test-error-drv";
-                  expected = null;
-                  actualDrv = pkgs.runCommand "test-error-drv" {} ''
-                    echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
-                    exit 1
-                  '';
-                }
-                {
-                  name = "test-script";
-                  type = "script";
-                  script = ''
-                    echo Test something here
-                    # required in pure mode:
-                    export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
-                    grep -q "test" ${builtins.toFile "test" "test"}
-                  '';
-                }
-              ];
-            };
-            "other-suite".tests = [
-              {
-                name = "obj-snapshot";
-                type = "snapshot";
-                pos = __curPos;
-                actual = {hello = "world";};
-              }
-              {
-                name = "pretty-snapshot";
-                type = "snapshot";
-                format = "pretty";
-                pos = __curPos;
-                actual = {
-                  example = args: {};
-                  example2 = {
-                    drv = pkgs.hello;
-                  };
-                };
-              }
-              {
-                name = "pretty-unit";
-                format = "pretty";
-                pos = __curPos;
-                expected = pkgs.hello;
-                actual = pkgs.hello;
-              }
-              {
-                name = "test-drv";
-                pos = __curPos;
-                expected = {a = "b";};
-                actualDrv = pkgs.runCommand "test-something" {} ''
-                  echo "Simulating taking some time"
-                  sleep 1
-                  echo '{"a":"b"}' > $out
-                '';
-              }
-              {
-                name = "skipped";
-                expected = null;
-                actual = null;
-              }
-            ];
-          };
-        };
-
         doc = {
           path = ./docs;
           deps = pp: [
@@ -315,10 +220,10 @@
           ntlib = import ./lib {inherit pkgs lib;};
         in {
           default = pkgs.callPackage ./package.nix {};
-          lib-tests = ntlib.mkNixtest {
-            modules = ntlib.autodiscover {dir = ./lib;};
+          tests = ntlib.mkNixtest {
+            modules = ntlib.autodiscover {dir = ./tests;};
             args = {
-              inherit pkgs;
+              inherit pkgs ntlib;
             };
           };
         };
diff --git a/lib/lib_test.nix b/lib/lib_test.nix
deleted file mode 100644
index 6a47538..0000000
--- a/lib/lib_test.nix
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-  pkgs,
-  lib,
-  ...
-}: let
-  ntlib = import ./. {inherit pkgs lib;};
-in {
-  suites."Lib Tests".tests = [
-    {
-      name = "autodiscovery";
-      type = "script";
-      script = let
-        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
-          dir = ./.;
-        }));
-      in
-        # sh
-        ''
-          export PATH="${pkgs.gnugrep}/bin"
-          grep -q lib_test.nix ${actual}
-          grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual}
-        '';
-    }
-    {
-      name = "binary";
-      type = "script";
-      script = let
-        binary =
-          (ntlib.mkBinary {
-            nixtests = "stub";
-            extraParams = "--pure";
-          })
-          + "/bin/nixtests:run";
-      in
-        # sh
-        ''
-          export PATH="${pkgs.gnugrep}/bin"
-          grep -q nixtest ${binary}
-          grep -q -- "--pure" ${binary}
-          grep -q -- "--tests=stub" ${binary}
-        '';
-    }
-  ];
-}
diff --git a/tests/fixtures/sample_test.nix b/tests/fixtures/sample_test.nix
new file mode 100644
index 0000000..dcc2fb1
--- /dev/null
+++ b/tests/fixtures/sample_test.nix
@@ -0,0 +1,97 @@
+{
+  lib,
+  pkgs,
+  ...
+}: {
+  skip = "skip.*d";
+  suites = {
+    "suite-one" = {
+      # required to figure out file and line, but optional
+      pos = __curPos;
+      tests = [
+        {
+          name = "test-one";
+          expected = 1;
+          actual = 1;
+        }
+        {
+          name = "fail";
+          expected = 0;
+          actual = "meow";
+        }
+        {
+          name = "snapshot-test";
+          type = "snapshot";
+          actual = "test";
+        }
+        {
+          name = "test-snapshot-drv";
+          type = "snapshot";
+          actualDrv = pkgs.runCommand "test-snapshot" {} ''
+            echo '"snapshot drv"' > $out
+          '';
+        }
+        {
+          name = "test-error-drv";
+          expected = null;
+          actualDrv = pkgs.runCommand "test-error-drv" {} ''
+            echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
+            exit 1
+          '';
+        }
+        {
+          name = "test-script";
+          type = "script";
+          script = ''
+            echo Test something here
+            # required in pure mode:
+            export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+            grep -q "test" ${builtins.toFile "test" "test"}
+          '';
+        }
+      ];
+    };
+    "other-suite".tests = [
+      {
+        name = "obj-snapshot";
+        type = "snapshot";
+        pos = __curPos;
+        actual = {hello = "world";};
+      }
+      {
+        name = "pretty-snapshot";
+        type = "snapshot";
+        format = "pretty";
+        pos = __curPos;
+        actual = {
+          example = args: {};
+          example2 = {
+            drv = pkgs.hello;
+          };
+        };
+      }
+      {
+        name = "pretty-unit";
+        format = "pretty";
+        pos = __curPos;
+        expected = pkgs.hello;
+        actual = pkgs.hello;
+      }
+      {
+        name = "test-drv";
+        pos = __curPos;
+        expected = {a = "b";};
+        actualDrv = pkgs.runCommand "test-something" {} ''
+          echo "Simulating taking some time"
+          sleep 1
+          echo '{"a":"b"}' > $out
+        '';
+      }
+      {
+        name = "skipped";
+        expected = null;
+        actual = null;
+      }
+    ];
+  };
+}
diff --git a/tests/lib_test.nix b/tests/lib_test.nix
new file mode 100644
index 0000000..bf21649
--- /dev/null
+++ b/tests/lib_test.nix
@@ -0,0 +1,86 @@
+{
+  lib,
+  pkgs,
+  ntlib,
+  ...
+}: {
+  suites."Lib Tests".tests = [
+    {
+      name = "autodiscovery";
+      type = "script";
+      script = let
+        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
+          dir = ./fixtures;
+        }));
+      in
+        # sh
+        ''
+          ${ntlib.helpers.path [pkgs.gnugrep]}
+          ${ntlib.helpers.scriptHelpers}
+          assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
+          assert_file_contains ${actual} "\"base\":\"/nix/store/.*-source/tests/fixtures" "should set base to fixtures dir"
+        '';
+    }
+    {
+      name = "binary";
+      type = "script";
+      script = let
+        binary =
+          (ntlib.mkBinary {
+            nixtests = "stub";
+            extraParams = "--pure";
+          })
+          + "/bin/nixtests:run";
+      in
+        # sh
+        ''
+          ${ntlib.helpers.path [pkgs.gnugrep]}
+          ${ntlib.helpers.scriptHelpers}
+          assert_file_contains ${binary} "nixtest" "should contain nixtest"
+          assert_file_contains ${binary} "--pure" "should contain --pure arg"
+          assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
+
+          run "${binary} --help"
+          assert_eq $exit_code 0 "should exit 0"
+          assert_contains "$output" "Usage of nixtest" "should show help"
+
+          run "${binary}"
+          assert_eq $exit_code 1 "should exit 1"
+          assert_contains "$output" "Tests file does not exist"
+        '';
+    }
+    {
+      name = "full run with fixtures";
+      type = "script";
+      script = let
+        binary =
+          (ntlib.mkNixtest {
+            modules = ntlib.autodiscover {dir = ./fixtures;};
+            args = {inherit pkgs;};
+          })
+          + "/bin/nixtests:run";
+      in
+        # sh
+        ''
+          ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
+          ${ntlib.helpers.scriptHelpers}
+
+          TMPDIR=$(tmpdir)
+          run "${binary} --pure --junit=$TMPDIR/junit.xml"
+          assert "$exit_code -eq 2" "should exit 2"
+          assert "-f $TMPDIR/junit.xml" "should create junit.xml"
+          assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
+
+          ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
+          run "${binary} --pure --junit=$TMPDIR/junit2.xml"
+          assert "$exit_code -eq 2" "should exit 2"
+          assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
+          assert_not_contains "$output" "executable file not found" "nix should now exist"
+          assert_contains "$output" "suite-one" "should contain suite-one"
+          assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
+          assert_contains "$output" "ERROR" "should contain an error"
+          assert_contains "$output" "SKIP" "should contain a skip"
+        '';
+    }
+  ];
+}

From 3f1b6317b4c43ab0e17c194f02c6a8ec138360fe Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Fri, 13 Jun 2025 15:48:19 +0200
Subject: [PATCH 5/8] ci: remove flakeModule test job

---
 flake.nix | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/flake.nix b/flake.nix
index da50a26..f0a2570 100644
--- a/flake.nix
+++ b/flake.nix
@@ -141,21 +141,10 @@
         ci = {
           stages = ["test" "build" "deploy"];
           jobs = {
-            "test:flakeModule" = {
-              stage = "test";
-              script = [
-                "nix run .#nixtests:run -- --junit=junit.xml"
-              ];
-              allow_failure = true;
-              artifacts = {
-                when = "always";
-                reports.junit = "junit.xml";
-              };
-            };
             "test:lib" = {
               stage = "test";
               script = [
-                "nix run .#lib-tests -- --junit=junit.xml"
+                "nix run .#tests -- --junit=junit.xml"
               ];
               allow_failure = true;
               artifacts = {

From 4a55db97979b8b06ebd68b4b0772fcb76d79d2fa Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Fri, 13 Jun 2025 20:47:26 +0200
Subject: [PATCH 6/8] chore: add more helpers & add pos to test suite

---
 lib/testHelpers.nix |   2 +
 tests/lib_test.nix  | 156 +++++++++++++++++++++++---------------------
 2 files changed, 82 insertions(+), 76 deletions(-)

diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix
index c29618d..0edb595 100644
--- a/lib/testHelpers.nix
+++ b/lib/testHelpers.nix
@@ -2,4 +2,6 @@
   path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
   pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
   scriptHelpers = builtins.readFile ./scriptHelpers.sh;
+  toJsonFile = any: builtins.toFile "actual" (builtins.toJSON any);
+  toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
 }
diff --git a/tests/lib_test.nix b/tests/lib_test.nix
index bf21649..4437cf3 100644
--- a/tests/lib_test.nix
+++ b/tests/lib_test.nix
@@ -1,86 +1,90 @@
 {
-  lib,
   pkgs,
   ntlib,
   ...
 }: {
-  suites."Lib Tests".tests = [
-    {
-      name = "autodiscovery";
-      type = "script";
-      script = let
-        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
-          dir = ./fixtures;
-        }));
-      in
-        # sh
-        ''
-          ${ntlib.helpers.path [pkgs.gnugrep]}
-          ${ntlib.helpers.scriptHelpers}
-          assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
-          assert_file_contains ${actual} "\"base\":\"/nix/store/.*-source/tests/fixtures" "should set base to fixtures dir"
-        '';
-    }
-    {
-      name = "binary";
-      type = "script";
-      script = let
-        binary =
-          (ntlib.mkBinary {
-            nixtests = "stub";
-            extraParams = "--pure";
-          })
-          + "/bin/nixtests:run";
-      in
-        # sh
-        ''
-          ${ntlib.helpers.path [pkgs.gnugrep]}
-          ${ntlib.helpers.scriptHelpers}
-          assert_file_contains ${binary} "nixtest" "should contain nixtest"
-          assert_file_contains ${binary} "--pure" "should contain --pure arg"
-          assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
+  suites."Lib Tests" = {
+    pos = __curPos;
+    tests = [
+      {
+        name = "autodiscovery";
+        type = "script";
+        script = let
+          actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover {
+            dir = ./fixtures;
+          });
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep]}
+            ${ntlib.helpers.scriptHelpers}
+            assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
+            assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir"
+          '';
+      }
+      {
+        name = "binary";
+        type = "script";
+        script = let
+          binary =
+            (ntlib.mkBinary {
+              nixtests = "stub";
+              extraParams = "--pure";
+            })
+            + "/bin/nixtests:run";
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep]}
+            ${ntlib.helpers.scriptHelpers}
+            assert_file_contains ${binary} "nixtest" "should contain nixtest"
+            assert_file_contains ${binary} "--pure" "should contain --pure arg"
+            assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
 
-          run "${binary} --help"
-          assert_eq $exit_code 0 "should exit 0"
-          assert_contains "$output" "Usage of nixtest" "should show help"
+            run "${binary} --help"
+            assert_eq $exit_code 0 "should exit 0"
+            assert_contains "$output" "Usage of nixtest" "should show help"
 
-          run "${binary}"
-          assert_eq $exit_code 1 "should exit 1"
-          assert_contains "$output" "Tests file does not exist"
-        '';
-    }
-    {
-      name = "full run with fixtures";
-      type = "script";
-      script = let
-        binary =
-          (ntlib.mkNixtest {
-            modules = ntlib.autodiscover {dir = ./fixtures;};
-            args = {inherit pkgs;};
-          })
-          + "/bin/nixtests:run";
-      in
-        # sh
-        ''
-          ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
-          ${ntlib.helpers.scriptHelpers}
+            run "${binary}"
+            assert_eq $exit_code 1 "should exit 1"
+            assert_contains "$output" "Tests file does not exist"
+          '';
+      }
+      {
+        name = "full run with fixtures";
+        type = "script";
+        script = let
+          binary =
+            (ntlib.mkNixtest {
+              modules = ntlib.autodiscover {dir = ./fixtures;};
+              args = {inherit pkgs;};
+            })
+            + "/bin/nixtests:run";
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
+            ${ntlib.helpers.scriptHelpers}
 
-          TMPDIR=$(tmpdir)
-          run "${binary} --pure --junit=$TMPDIR/junit.xml"
-          assert "$exit_code -eq 2" "should exit 2"
-          assert "-f $TMPDIR/junit.xml" "should create junit.xml"
-          assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
+            TMPDIR=$(tmpdir)
+            # start without nix & env binaries to expect errors
+            run "${binary} --pure --junit=$TMPDIR/junit.xml"
+            assert "$exit_code -eq 2" "should exit 2"
+            assert "-f $TMPDIR/junit.xml" "should create junit.xml"
+            assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
 
-          ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
-          run "${binary} --pure --junit=$TMPDIR/junit2.xml"
-          assert "$exit_code -eq 2" "should exit 2"
-          assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
-          assert_not_contains "$output" "executable file not found" "nix should now exist"
-          assert_contains "$output" "suite-one" "should contain suite-one"
-          assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
-          assert_contains "$output" "ERROR" "should contain an error"
-          assert_contains "$output" "SKIP" "should contain a skip"
-        '';
-    }
-  ];
+            # now add required deps
+            ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
+            run "${binary} --pure --junit=$TMPDIR/junit2.xml"
+            assert "$exit_code -eq 2" "should exit 2"
+            assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
+            assert_not_contains "$output" "executable file not found" "nix should now exist"
+            assert_contains "$output" "suite-one" "should contain suite-one"
+            assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
+            assert_contains "$output" "ERROR" "should contain an error"
+            assert_contains "$output" "SKIP" "should contain a skip"
+          '';
+      }
+    ];
+  };
 }

From 116f905b6c1df01f7c2dddef4f662abd0fcb5d42 Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Sat, 14 Jun 2025 15:16:25 +0200
Subject: [PATCH 7/8] chore(testHelpers): remove string context in toJsonFile

---
 lib/testHelpers.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix
index 0edb595..c86a6a8 100644
--- a/lib/testHelpers.nix
+++ b/lib/testHelpers.nix
@@ -2,6 +2,6 @@
   path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
   pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
   scriptHelpers = builtins.readFile ./scriptHelpers.sh;
-  toJsonFile = any: builtins.toFile "actual" (builtins.toJSON any);
+  toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any));
   toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
 }

From bc36c39b0929bdfaab0908d8bd852c2114e3383f Mon Sep 17 00:00:00 2001
From: technofab <admin@technofab.de>
Date: Sat, 14 Jun 2025 17:23:24 +0200
Subject: [PATCH 8/8] docs: document new lib functions & usage

---
 docs/examples.md  |  4 +++-
 docs/reference.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++
 docs/usage.md     | 40 ++++++++++++++++++++++++++++++-----
 flake.nix         |  2 ++
 4 files changed, 93 insertions(+), 6 deletions(-)
 create mode 100644 docs/reference.md

diff --git a/docs/examples.md b/docs/examples.md
index 4e3ad41..c61c754 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -1,4 +1,6 @@
 # Example Configs
 
+- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest)
+    see `flake.nix` and `tests/`
 - [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci)
-    see tests/
+    see `tests/`
diff --git a/docs/reference.md b/docs/reference.md
new file mode 100644
index 0000000..fe9d7cb
--- /dev/null
+++ b/docs/reference.md
@@ -0,0 +1,53 @@
+# Reference
+
+## `flakeModule`
+
+The `flakeModule` for [flake-parts](https://flake.parts).
+
+## `lib`
+
+### `module`
+
+The nix module for validation of inputs etc.
+Used internally by `mkNixtestConfig`.
+
+### `autodiscover`
+
+```nix
+autodiscover {
+  dir,
+  pattern ? ".*_test.nix",
+}
+```
+
+Finds all test files in `dir` matching `pattern`.
+Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg).
+
+### `mkNixtestConfig`
+
+```nix
+mkNixtestConfig {
+  modules,
+  args ? {},
+}
+```
+
+Evaluates the test `modules`.
+`args` are passed to the modules using `_module.args = args`.
+
+**Noteworthy attributes**:
+
+- `app`: nixtest wrapper
+- `finalConfigJson`: derivation containing the tests json file
+
+### `mkNixtest`
+
+```nix
+mkNixtest {
+  modules,
+  args ? {},
+}
+```
+
+Creates the nixtest wrapper, using the tests in `modules`.
+Basically `(mkNixtestConfig <arguments>).app`.
diff --git a/docs/usage.md b/docs/usage.md
index fbcccb3..a329017 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -2,6 +2,9 @@
 
 ## Flake Module
 
+The easiest way to use Nixtest is probably using the flakeModule.
+Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`:
+
 ```nix
 {
   inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib";
@@ -34,10 +37,25 @@
 
 ## Library
 
-You can also integrate nixtest in your own workflow by using the lib functions directly.
-Check out `flakeModule.nix` to see how it's used there.
+You can also use the lib directly, like this for example:
 
-<!-- TODO: more detailed? -->
+```nix
+packages.tests = ntlib.mkNixtest {
+  modules = ntlib.autodiscover {dir = ./tests;};
+  args = {
+    inherit pkgs ntlib;
+  };
+};
+```
+
+This will auto-discover all test files ending with `_test.nix`.
+See [reference](reference.md) for all params to `autodiscover`.
+
+`ntlib` can be defined like this:
+
+```nix
+ntlib = inputs.nixtests.lib {inherit pkgs;};
+```
 
 ## Define Tests
 
@@ -84,9 +102,17 @@ Examples:
     # to make it more reproducible and cleaner, use --pure to switch to pure
     # mode which will unset all env variables before running the test. That 
     # requires you to set PATH yourself then:
+    # 
+    # ''
+    #  export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+    #  grep -q "test" ${builtins.toFile "test" "test"}
+    # '';
+    #
+    # you can also use the helpers to make it nicer to read:
     ''
-      export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
-      grep -q "test" ${builtins.toFile "test" "test"}
+      ${ntlib.helpers.path [pkgs.gnugrep]}
+      ${ntlib.helpers.scriptHelpers}  # this adds helpers like assert etc.
+      assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'"
     '';
   }
   {
@@ -102,3 +128,7 @@ Examples:
   }
 ]
 ```
+
+!!! note
+
+    for more examples see [examples](./examples.md)
diff --git a/flake.nix b/flake.nix
index f0a2570..72f86bd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -106,11 +106,13 @@
             nav = [
               {"Introduction" = "index.md";}
               {"Usage" = "usage.md";}
+              {"Reference" = "reference.md";}
               {"CLI" = "cli.md";}
               {"Example Configs" = "examples.md";}
             ];
             markdown_extensions = [
               "pymdownx.superfences"
+              "admonition"
             ];
             extra.analytics = {
               provider = "umami";