feat: switch to module system to evaluate suites & tests

add autodiscovery feature

flake.nix

@@ -236,7 +236,7 @@
         ci = {
           stages = ["test" "build" "deploy"];
           jobs = {
-            "test" = {
+            "test:flakeModule" = {
               stage = "test";
               script = [
                 "nix run .#nixtests:run -- --junit=junit.xml"
@@ -247,6 +247,17 @@
                 reports.junit = "junit.xml";
               };
             };
+            "test:lib" = {
+              stage = "test";
+              script = [
+                "nix run .#lib-tests -- --junit=junit.xml"
+              ];
+              allow_failure = true;
+              artifacts = {
+                when = "always";
+                reports.junit = "junit.xml";
+              };
+            };
             "test:go" = {
               stage = "test";
               nix.deps = with pkgs; [go go-junit-report gocover-cobertura];
@@ -300,7 +311,17 @@
           };
         };
 
-        packages.default = pkgs.callPackage ./package.nix {};
+        packages = let
+          ntlib = import ./lib {inherit pkgs lib;};
+        in {
+          default = pkgs.callPackage ./package.nix {};
+          lib-tests = ntlib.mkNixtest {
+            modules = ntlib.autodiscover {dir = ./lib;};
+            args = {
+              inherit pkgs;
+            };
+          };
+        };
       };
     };
 

lib/default.nix

@@ -1,62 +1,64 @@
 {
   pkgs,
   lib ? pkgs.lib,
-  self ? "",
   ...
-}: {
+}: let
-  mkTest = {
+  inherit (lib) evalModules toList;
-    type ? "unit",
+in rec {
-    name,
+  mkBinary = {
-    description ? "",
+    nixtests,
-    format ? "json",
+    extraParams,
-    expected ? null,
-    actual ? null,
-    actualDrv ? null,
-    script ? null,
-    pos ? null,
   }: let
-    fileRelative = lib.removePrefix ((toString self) + "/") pos.file;
+    program = pkgs.callPackage ../package.nix {};
-    actual' =
-      if format == "json"
-      then actual
-      else lib.generators.toPretty {} actual;
-    expected' =
-      if format == "json"
-      then expected
-      else lib.generators.toPretty {} expected;
   in
-    assert lib.assertMsg (!(type == "script" && script == null)) "test ${name} has type 'script' but no script was passed"; {
+    (pkgs.writeShellScriptBin "nixtests:run" ''
-      inherit type name description;
+      ${program}/bin/nixtest --tests=${nixtests} ${extraParams} "$@"
-      actual = actual';
+    '')
-      expected = expected';
+    // {
-      # discard string context, otherwise it's being built instantly which we don't want
+      tests = nixtests;
-      actualDrv = builtins.unsafeDiscardStringContext (actualDrv.drvPath or "");
-      script =
-        if script != null
-        then
-          builtins.unsafeDiscardStringContext
-          (pkgs.writeShellScript "nixtest-${name}" ''
-            # show which line failed the test
-            set -x
-            ${script}
-          '').drvPath
-        else null;
-      pos =
-        if pos == null
-        then ""
-        else "${fileRelative}:${toString pos.line}";
     };
-  mkSuite = name: tests: {
+
-    inherit name tests;
-  };
   exportSuites = suites: let
     suitesList =
       if builtins.isList suites
       then suites
       else [suites];
-    testsMapped = builtins.toJSON suitesList;
+    suitesMapped = builtins.toJSON suitesList;
   in
     pkgs.runCommand "tests.json" {} ''
-      echo '${testsMapped}' > $out
+      echo '${suitesMapped}' > $out
     '';
+
+  module = import ./module.nix {inherit lib pkgs;};
+
+  autodiscover = {
+    dir,
+    pattern ? ".*_test.nix",
+  }: let
+    files = builtins.readDir dir;
+    matchingFiles = builtins.filter (name: builtins.match pattern name != null) (builtins.attrNames files);
+    imports = map (file: /${dir}/${file}) matchingFiles;
+  in {
+    inherit imports;
+    # automatically set the base so test filepaths are easier to read
+    config.base = builtins.toString dir + "/";
+  };
+
+  mkNixtestConfig = {
+    modules,
+    args,
+    ...
+  }:
+    (evalModules {
+      modules =
+        (toList modules)
+        ++ [
+          module
+          {
+            _module.args = args;
+          }
+        ];
+    }).config;
+
+  mkNixtest = args: (mkNixtestConfig args).app;
 }

lib/flakeModule.nix

@@ -15,56 +15,14 @@ in {
       nixtests-lib = import ./. {inherit pkgs self;};
     in {
       options.nixtest = mkOption {
-        type = types.submodule ({...}: {
+        type = types.submodule (nixtests-lib.module);
-          options = {
-            skip = mkOption {
-              type = types.str;
-              default = "";
-              description = "Which tests to skip (regex)";
-            };
-            suites = mkOption {
-              type = types.attrsOf (types.submodule {
-                options = {
-                  tests = mkOption {
-                    type = types.listOf types.attrs;
-                    default = [];
-                  };
-                  pos = mkOption {
-                    type = types.nullOr types.attrs;
-                    default = null;
-                  };
-                };
-              });
-              default = {};
-            };
-          };
-        });
         default = {};
       };
+      config.nixtest.base = toString self + "/";
 
-      config.legacyPackages = rec {
+      config.legacyPackages = {
-        "nixtests" = let
+        "nixtests" = config.nixtest.finalConfigJson;
-          suites = map (suiteName: let
+        "nixtests:run" = config.nixtest.app;
-            suite = builtins.getAttr suiteName config.nixtest.suites;
-          in
-            nixtests-lib.mkSuite
-            suiteName
-            (map (test:
-              nixtests-lib.mkTest ({
-                  # default pos to suite's pos if given
-                  pos = suite.pos;
-                }
-                // test))
-            suite.tests))
-          (builtins.attrNames config.nixtest.suites);
-        in
-          nixtests-lib.exportSuites suites;
-        "nixtests:run" = let
-          program = pkgs.callPackage ./../package.nix {};
-        in
-          pkgs.writeShellScriptBin "nixtests:run" ''
-            ${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
-          '';
       };
     }
   );

lib/lib_test.nix

@@ -0,0 +1,44 @@
+{
+  pkgs,
+  lib,
+  ...
+}: let
+  ntlib = import ./. {inherit pkgs lib;};
+in {
+  suites."Lib Tests".tests = [
+    {
+      name = "autodiscovery";
+      type = "script";
+      script = let
+        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
+          dir = ./.;
+        }));
+      in
+        # sh
+        ''
+          export PATH="${pkgs.gnugrep}/bin"
+          grep -q lib_test.nix ${actual}
+          grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual}
+        '';
+    }
+    {
+      name = "binary";
+      type = "script";
+      script = let
+        binary =
+          (ntlib.mkBinary {
+            nixtests = "stub";
+            extraParams = "--pure";
+          })
+          + "/bin/nixtests:run";
+      in
+        # sh
+        ''
+          export PATH="${pkgs.gnugrep}/bin"
+          grep -q nixtest ${binary}
+          grep -q -- "--pure" ${binary}
+          grep -q -- "--tests=stub" ${binary}
+        '';
+    }
+  ];
+}

lib/module.nix

@@ -0,0 +1,192 @@
+{
+  pkgs,
+  lib,
+  ...
+}: let
+  inherit (lib) mkOptionType mkOption types;
+
+  nixtest-lib = import ./default.nix {inherit pkgs lib;};
+
+  unsetType = mkOptionType {
+    name = "unset";
+    description = "unset";
+    descriptionClass = "noun";
+    check = value: true;
+  };
+  unset = {
+    _type = "unset";
+  };
+  isUnset = lib.isType "unset";
+
+  filterUnset = value:
+    if builtins.isAttrs value && !builtins.hasAttr "_type" value
+    then let
+      filteredAttrs = builtins.mapAttrs (n: v: filterUnset v) value;
+    in
+      lib.filterAttrs (name: value: (!isUnset value)) filteredAttrs
+    else if builtins.isList value
+    then builtins.filter (elem: !isUnset elem) (map filterUnset value)
+    else value;
+
+  testsSubmodule = {
+    config,
+    testsBase,
+    pos,
+    ...
+  }: {
+    options = {
+      pos = mkOption {
+        type = types.either types.attrs unsetType;
+        default = pos;
+        apply = val:
+          if isUnset val
+          then val
+          else let
+            fileRelative = lib.removePrefix testsBase val.file;
+          in "${fileRelative}:${toString val.line}";
+      };
+      type = mkOption {
+        type = types.enum ["unit" "snapshot" "script"];
+        default = "unit";
+        apply = value:
+          assert lib.assertMsg (value != "script" || !isUnset config.script)
+          "test '${config.name}' as type 'script' requires 'script' to be set";
+          assert lib.assertMsg (value != "unit" || !isUnset config.expected)
+          "test '${config.name}' as type 'unit' requires 'expected' to be set";
+          assert lib.assertMsg (
+            let
+              actualIsUnset = isUnset config.actual;
+              actualDrvIsUnset = isUnset config.actualDrv;
+            in
+              (value != "unit")
+              || (!actualIsUnset && actualDrvIsUnset)
+              || (actualIsUnset && !actualDrvIsUnset)
+          )
+          "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value;
+      };
+      name = mkOption {
+        type = types.str;
+      };
+      description = mkOption {
+        type = types.either types.str unsetType;
+        default = unset;
+      };
+      format = mkOption {
+        type = types.enum ["json" "pretty"];
+        default = "json";
+      };
+      expected = mkOption {
+        type = types.anything;
+        default = unset;
+        apply = val:
+          if isUnset val || config.format == "json"
+          then val
+          else lib.generators.toPretty {} val;
+      };
+      actual = mkOption {
+        type = types.anything;
+        default = unset;
+        apply = val:
+          if isUnset val || config.format == "json"
+          then val
+          else lib.generators.toPretty {} val;
+      };
+      actualDrv = mkOption {
+        type = types.either types.package unsetType;
+        default = unset;
+        apply = val:
+        # keep unset value
+          if isUnset val
+          then val
+          else builtins.unsafeDiscardStringContext (val.drvPath or "");
+      };
+      script = mkOption {
+        type = types.either types.str unsetType;
+        default = unset;
+        apply = val:
+          if isUnset val
+          then val
+          else
+            builtins.unsafeDiscardStringContext
+            (pkgs.writeShellScript "nixtest-${config.name}" ''
+              # show which line failed the test
+              set -x
+              ${val}
+            '').drvPath;
+      };
+    };
+  };
+
+  suitesSubmodule = {
+    name,
+    config,
+    testsBase,
+    ...
+  }: {
+    options = {
+      name = mkOption {
+        type = types.str;
+        default = name;
+      };
+      pos = mkOption {
+        type = types.either types.attrs unsetType;
+        default = unset;
+      };
+      tests = mkOption {
+        type = types.listOf (types.submoduleWith {
+          modules = [testsSubmodule];
+          specialArgs = {
+            inherit (config) pos;
+            inherit testsBase;
+          };
+        });
+        default = [];
+      };
+    };
+  };
+
+  nixtestSubmodule = {config, ...}: {
+    options = {
+      base = mkOption {
+        description = "Base directory of the tests, will be removed from the test file path";
+        type = types.str;
+        default = "";
+      };
+      skip = mkOption {
+        type = types.str;
+        default = "";
+      };
+      suites = mkOption {
+        type = types.attrsOf (types.submoduleWith {
+          modules = [suitesSubmodule];
+          specialArgs = {
+            testsBase = config.base;
+          };
+        });
+        default = {};
+        apply = suites:
+          map (
+            n: filterUnset (builtins.removeAttrs suites.${n} ["pos"])
+          )
+          (builtins.attrNames suites);
+      };
+
+      finalConfigJson = mkOption {
+        internal = true;
+        type = types.package;
+      };
+      app = mkOption {
+        internal = true;
+        type = types.package;
+      };
+    };
+    config = {
+      finalConfigJson = nixtest-lib.exportSuites config.suites;
+      app = nixtest-lib.mkBinary {
+        nixtests = config.finalConfigJson;
+        extraParams = ''--skip="${config.skip}"'';
+      };
+    };
+  };
+in
+  nixtestSubmodule