Merge branch 'feat/module-system' into 'main'

Draft: feat: switch to module system to evaluate suites & tests

Closes #8

See merge request TECHNOFAB/nixtest!2

docs/examples.md

@@ -1,4 +1,6 @@
 # Example Configs
 
+- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest)
+    see `flake.nix` and `tests/`
 - [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci)
-    see tests/
+    see `tests/`

docs/reference.md

@@ -0,0 +1,53 @@
+# Reference
+
+## `flakeModule`
+
+The `flakeModule` for [flake-parts](https://flake.parts).
+
+## `lib`
+
+### `module`
+
+The nix module for validation of inputs etc.
+Used internally by `mkNixtestConfig`.
+
+### `autodiscover`
+
+```nix
+autodiscover {
+  dir,
+  pattern ? ".*_test.nix",
+}
+```
+
+Finds all test files in `dir` matching `pattern`.
+Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg).
+
+### `mkNixtestConfig`
+
+```nix
+mkNixtestConfig {
+  modules,
+  args ? {},
+}
+```
+
+Evaluates the test `modules`.
+`args` are passed to the modules using `_module.args = args`.
+
+**Noteworthy attributes**:
+
+- `app`: nixtest wrapper
+- `finalConfigJson`: derivation containing the tests json file
+
+### `mkNixtest`
+
+```nix
+mkNixtest {
+  modules,
+  args ? {},
+}
+```
+
+Creates the nixtest wrapper, using the tests in `modules`.
+Basically `(mkNixtestConfig <arguments>).app`.

docs/usage.md

@@ -2,6 +2,9 @@
 
 ## Flake Module
 
+The easiest way to use Nixtest is probably using the flakeModule.
+Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`:
+
 ```nix
 {
   inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib";
@@ -34,10 +37,25 @@
 
 ## Library
 
-You can also integrate nixtest in your own workflow by using the lib functions directly.
+You can also use the lib directly, like this for example:
-Check out `flakeModule.nix` to see how it's used there.
 
-<!-- TODO: more detailed? -->
+```nix
+packages.tests = ntlib.mkNixtest {
+  modules = ntlib.autodiscover {dir = ./tests;};
+  args = {
+    inherit pkgs ntlib;
+  };
+};
+```
+
+This will auto-discover all test files ending with `_test.nix`.
+See [reference](reference.md) for all params to `autodiscover`.
+
+`ntlib` can be defined like this:
+
+```nix
+ntlib = inputs.nixtests.lib {inherit pkgs;};
+```
 
 ## Define Tests
 
@@ -84,9 +102,17 @@ Examples:
     # to make it more reproducible and cleaner, use --pure to switch to pure
     # mode which will unset all env variables before running the test. That 
     # requires you to set PATH yourself then:
+    # 
+    # ''
+    #  export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+    #  grep -q "test" ${builtins.toFile "test" "test"}
+    # '';
+    #
+    # you can also use the helpers to make it nicer to read:
     ''
-      export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+      ${ntlib.helpers.path [pkgs.gnugrep]}
-      grep -q "test" ${builtins.toFile "test" "test"}
+      ${ntlib.helpers.scriptHelpers}  # this adds helpers like assert etc.
+      assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'"
     '';
   }
   {
@@ -102,3 +128,7 @@ Examples:
   }
 ]
 ```
+
+!!! note
+
+    for more examples see [examples](./examples.md)

flake.nix

@@ -11,7 +11,6 @@
         inputs.nix-gitlab-ci.flakeModule
         inputs.nix-devtools.flakeModule
         inputs.nix-mkdocs.flakeModule
-        ./lib/flakeModule.nix
       ];
       systems = import systems;
       flake = {};
@@ -63,100 +62,6 @@
           };
         };
 
-        nixtest = {
-          skip = "skip.*d";
-          suites = {
-            "suite-one" = {
-              pos = __curPos;
-              tests = [
-                {
-                  name = "test-one";
-                  # required to figure out file and line, but optional
-                  expected = 1;
-                  actual = 1;
-                }
-                {
-                  name = "fail";
-                  expected = 0;
-                  actual = "meow";
-                }
-                {
-                  name = "snapshot-test";
-                  type = "snapshot";
-                  actual = "test";
-                }
-                {
-                  name = "test-snapshot-drv";
-                  type = "snapshot";
-                  actualDrv = pkgs.runCommand "test-snapshot" {} ''
-                    echo '"snapshot drv"' > $out
-                  '';
-                }
-                {
-                  name = "test-error-drv";
-                  expected = null;
-                  actualDrv = pkgs.runCommand "test-error-drv" {} ''
-                    echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
-                    exit 1
-                  '';
-                }
-                {
-                  name = "test-script";
-                  type = "script";
-                  script = ''
-                    echo Test something here
-                    # required in pure mode:
-                    export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
-                    grep -q "test" ${builtins.toFile "test" "test"}
-                  '';
-                }
-              ];
-            };
-            "other-suite".tests = [
-              {
-                name = "obj-snapshot";
-                type = "snapshot";
-                pos = __curPos;
-                actual = {hello = "world";};
-              }
-              {
-                name = "pretty-snapshot";
-                type = "snapshot";
-                format = "pretty";
-                pos = __curPos;
-                actual = {
-                  example = args: {};
-                  example2 = {
-                    drv = pkgs.hello;
-                  };
-                };
-              }
-              {
-                name = "pretty-unit";
-                format = "pretty";
-                pos = __curPos;
-                expected = pkgs.hello;
-                actual = pkgs.hello;
-              }
-              {
-                name = "test-drv";
-                pos = __curPos;
-                expected = {a = "b";};
-                actualDrv = pkgs.runCommand "test-something" {} ''
-                  echo "Simulating taking some time"
-                  sleep 1
-                  echo '{"a":"b"}' > $out
-                '';
-              }
-              {
-                name = "skipped";
-                expected = null;
-                actual = null;
-              }
-            ];
-          };
-        };
-
         doc = {
           path = ./docs;
           deps = pp: [
@@ -201,11 +106,13 @@
             nav = [
               {"Introduction" = "index.md";}
               {"Usage" = "usage.md";}
+              {"Reference" = "reference.md";}
               {"CLI" = "cli.md";}
               {"Example Configs" = "examples.md";}
             ];
             markdown_extensions = [
               "pymdownx.superfences"
+              "admonition"
             ];
             extra.analytics = {
               provider = "umami";
@@ -236,21 +143,10 @@
         ci = {
           stages = ["test" "build" "deploy"];
           jobs = {
-            "test:flakeModule" = {
-              stage = "test";
-              script = [
-                "nix run .#nixtests:run -- --junit=junit.xml"
-              ];
-              allow_failure = true;
-              artifacts = {
-                when = "always";
-                reports.junit = "junit.xml";
-              };
-            };
             "test:lib" = {
               stage = "test";
               script = [
-                "nix run .#lib-tests -- --junit=junit.xml"
+                "nix run .#tests -- --junit=junit.xml"
               ];
               allow_failure = true;
               artifacts = {
@@ -315,10 +211,10 @@
           ntlib = import ./lib {inherit pkgs lib;};
         in {
           default = pkgs.callPackage ./package.nix {};
-          lib-tests = ntlib.mkNixtest {
+          tests = ntlib.mkNixtest {
-            modules = ntlib.autodiscover {dir = ./lib;};
+            modules = ntlib.autodiscover {dir = ./tests;};
             args = {
-              inherit pkgs;
+              inherit pkgs ntlib;
             };
           };
         };

internal/config/config.go

@@ -1,6 +1,9 @@
 package config
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/jedib0t/go-pretty/v6/text"
 	"github.com/rs/zerolog/log"
 	flag "github.com/spf13/pflag"
@@ -28,9 +31,16 @@ func Load() AppConfig {
 	flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')")
 	flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests")
 	flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring")
+	helpRequested := flag.BoolP("help", "h", false, "Show this menu")
 
 	flag.Parse()
 
+	if *helpRequested {
+		fmt.Println("Usage of nixtest:")
+		flag.PrintDefaults()
+		os.Exit(0)
+	}
+
 	if cfg.TestsFile == "" {
 		log.Panic().Msg("Tests file path (-f or --tests) is required.")
 	}

lib/default.nix

@@ -5,6 +5,8 @@
 }: let
   inherit (lib) evalModules toList;
 in rec {
+  helpers = import ./testHelpers.nix {inherit lib;};
+
   mkBinary = {
     nixtests,
     extraParams,
@@ -46,7 +48,7 @@ in rec {
 
   mkNixtestConfig = {
     modules,
-    args,
+    args ? {},
     ...
   }:
     (evalModules {

lib/lib_test.nix

@@ -1,44 +0,0 @@
-{
-  pkgs,
-  lib,
-  ...
-}: let
-  ntlib = import ./. {inherit pkgs lib;};
-in {
-  suites."Lib Tests".tests = [
-    {
-      name = "autodiscovery";
-      type = "script";
-      script = let
-        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
-          dir = ./.;
-        }));
-      in
-        # sh
-        ''
-          export PATH="${pkgs.gnugrep}/bin"
-          grep -q lib_test.nix ${actual}
-          grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual}
-        '';
-    }
-    {
-      name = "binary";
-      type = "script";
-      script = let
-        binary =
-          (ntlib.mkBinary {
-            nixtests = "stub";
-            extraParams = "--pure";
-          })
-          + "/bin/nixtests:run";
-      in
-        # sh
-        ''
-          export PATH="${pkgs.gnugrep}/bin"
-          grep -q nixtest ${binary}
-          grep -q -- "--pure" ${binary}
-          grep -q -- "--tests=stub" ${binary}
-        '';
-    }
-  ];
-}

lib/module.nix

@@ -108,11 +108,7 @@
           then val
           else
             builtins.unsafeDiscardStringContext
-            (pkgs.writeShellScript "nixtest-${config.name}" ''
+            (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath;
-              # show which line failed the test
-              set -x
-              ${val}
-            '').drvPath;
       };
     };
   };

lib/scriptHelpers.sh

@@ -0,0 +1,51 @@
+output=
+exit_code=
+
+function assert() {
+  test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; }
+}
+function assert_eq() {
+  assert "$1 -eq $2" "$3"
+}
+function assert_not_eq() {
+  assert "$1 -ne $2" "$3"
+}
+function assert_contains() {
+  echo "$1" | grep -q -- "$2" || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_not_contains() {
+  echo "$1" | grep -q -- "$2" && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_contains() {
+  grep -q -- "$2" $1 || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_not_contains() {
+  grep -q -- "$2" $1 && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+
+function tmpdir() {
+  dir=$(mktemp -d)
+  trap "rm -rf $dir" EXIT
+  echo -n "$dir"
+}
+function tmpfile() {
+  file=$(mktemp)
+  trap "rm -f $file" EXIT
+  echo -n "$file"
+}
+function run() {
+  output=$($@ 2>&1)
+  exit_code=$?
+}

lib/testHelpers.nix

@@ -0,0 +1,7 @@
+{lib, ...}: {
+  path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
+  pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
+  scriptHelpers = builtins.readFile ./scriptHelpers.sh;
+  toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any));
+  toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
+}

tests/fixtures/sample_test.nix

@@ -0,0 +1,97 @@
+{
+  lib,
+  pkgs,
+  ...
+}: {
+  skip = "skip.*d";
+  suites = {
+    "suite-one" = {
+      # required to figure out file and line, but optional
+      pos = __curPos;
+      tests = [
+        {
+          name = "test-one";
+          expected = 1;
+          actual = 1;
+        }
+        {
+          name = "fail";
+          expected = 0;
+          actual = "meow";
+        }
+        {
+          name = "snapshot-test";
+          type = "snapshot";
+          actual = "test";
+        }
+        {
+          name = "test-snapshot-drv";
+          type = "snapshot";
+          actualDrv = pkgs.runCommand "test-snapshot" {} ''
+            echo '"snapshot drv"' > $out
+          '';
+        }
+        {
+          name = "test-error-drv";
+          expected = null;
+          actualDrv = pkgs.runCommand "test-error-drv" {} ''
+            echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
+            exit 1
+          '';
+        }
+        {
+          name = "test-script";
+          type = "script";
+          script = ''
+            echo Test something here
+            # required in pure mode:
+            export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+            grep -q "test" ${builtins.toFile "test" "test"}
+          '';
+        }
+      ];
+    };
+    "other-suite".tests = [
+      {
+        name = "obj-snapshot";
+        type = "snapshot";
+        pos = __curPos;
+        actual = {hello = "world";};
+      }
+      {
+        name = "pretty-snapshot";
+        type = "snapshot";
+        format = "pretty";
+        pos = __curPos;
+        actual = {
+          example = args: {};
+          example2 = {
+            drv = pkgs.hello;
+          };
+        };
+      }
+      {
+        name = "pretty-unit";
+        format = "pretty";
+        pos = __curPos;
+        expected = pkgs.hello;
+        actual = pkgs.hello;
+      }
+      {
+        name = "test-drv";
+        pos = __curPos;
+        expected = {a = "b";};
+        actualDrv = pkgs.runCommand "test-something" {} ''
+          echo "Simulating taking some time"
+          sleep 1
+          echo '{"a":"b"}' > $out
+        '';
+      }
+      {
+        name = "skipped";
+        expected = null;
+        actual = null;
+      }
+    ];
+  };
+}

tests/lib_test.nix

@@ -0,0 +1,90 @@
+{
+  pkgs,
+  ntlib,
+  ...
+}: {
+  suites."Lib Tests" = {
+    pos = __curPos;
+    tests = [
+      {
+        name = "autodiscovery";
+        type = "script";
+        script = let
+          actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover {
+            dir = ./fixtures;
+          });
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep]}
+            ${ntlib.helpers.scriptHelpers}
+            assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
+            assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir"
+          '';
+      }
+      {
+        name = "binary";
+        type = "script";
+        script = let
+          binary =
+            (ntlib.mkBinary {
+              nixtests = "stub";
+              extraParams = "--pure";
+            })
+            + "/bin/nixtests:run";
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep]}
+            ${ntlib.helpers.scriptHelpers}
+            assert_file_contains ${binary} "nixtest" "should contain nixtest"
+            assert_file_contains ${binary} "--pure" "should contain --pure arg"
+            assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
+
+            run "${binary} --help"
+            assert_eq $exit_code 0 "should exit 0"
+            assert_contains "$output" "Usage of nixtest" "should show help"
+
+            run "${binary}"
+            assert_eq $exit_code 1 "should exit 1"
+            assert_contains "$output" "Tests file does not exist"
+          '';
+      }
+      {
+        name = "full run with fixtures";
+        type = "script";
+        script = let
+          binary =
+            (ntlib.mkNixtest {
+              modules = ntlib.autodiscover {dir = ./fixtures;};
+              args = {inherit pkgs;};
+            })
+            + "/bin/nixtests:run";
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
+            ${ntlib.helpers.scriptHelpers}
+
+            TMPDIR=$(tmpdir)
+            # start without nix & env binaries to expect errors
+            run "${binary} --pure --junit=$TMPDIR/junit.xml"
+            assert "$exit_code -eq 2" "should exit 2"
+            assert "-f $TMPDIR/junit.xml" "should create junit.xml"
+            assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
+
+            # now add required deps
+            ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
+            run "${binary} --pure --junit=$TMPDIR/junit2.xml"
+            assert "$exit_code -eq 2" "should exit 2"
+            assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
+            assert_not_contains "$output" "executable file not found" "nix should now exist"
+            assert_contains "$output" "suite-one" "should contain suite-one"
+            assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
+            assert_contains "$output" "ERROR" "should contain an error"
+            assert_contains "$output" "SKIP" "should contain a skip"
+          '';
+      }
+    ];
+  };
+}