ci: remove flakeModule test job

also use new test helpers

flake.nix

@@ -11,7 +11,6 @@
         inputs.nix-gitlab-ci.flakeModule
         inputs.nix-devtools.flakeModule
         inputs.nix-mkdocs.flakeModule
-        ./lib/flakeModule.nix
       ];
       systems = import systems;
       flake = {};
@@ -63,100 +62,6 @@
           };
         };
 
-        nixtest = {
-          skip = "skip.*d";
-          suites = {
-            "suite-one" = {
-              pos = __curPos;
-              tests = [
-                {
-                  name = "test-one";
-                  # required to figure out file and line, but optional
-                  expected = 1;
-                  actual = 1;
-                }
-                {
-                  name = "fail";
-                  expected = 0;
-                  actual = "meow";
-                }
-                {
-                  name = "snapshot-test";
-                  type = "snapshot";
-                  actual = "test";
-                }
-                {
-                  name = "test-snapshot-drv";
-                  type = "snapshot";
-                  actualDrv = pkgs.runCommand "test-snapshot" {} ''
-                    echo '"snapshot drv"' > $out
-                  '';
-                }
-                {
-                  name = "test-error-drv";
-                  expected = null;
-                  actualDrv = pkgs.runCommand "test-error-drv" {} ''
-                    echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
-                    exit 1
-                  '';
-                }
-                {
-                  name = "test-script";
-                  type = "script";
-                  script = ''
-                    echo Test something here
-                    # required in pure mode:
-                    export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
-                    grep -q "test" ${builtins.toFile "test" "test"}
-                  '';
-                }
-              ];
-            };
-            "other-suite".tests = [
-              {
-                name = "obj-snapshot";
-                type = "snapshot";
-                pos = __curPos;
-                actual = {hello = "world";};
-              }
-              {
-                name = "pretty-snapshot";
-                type = "snapshot";
-                format = "pretty";
-                pos = __curPos;
-                actual = {
-                  example = args: {};
-                  example2 = {
-                    drv = pkgs.hello;
-                  };
-                };
-              }
-              {
-                name = "pretty-unit";
-                format = "pretty";
-                pos = __curPos;
-                expected = pkgs.hello;
-                actual = pkgs.hello;
-              }
-              {
-                name = "test-drv";
-                pos = __curPos;
-                expected = {a = "b";};
-                actualDrv = pkgs.runCommand "test-something" {} ''
-                  echo "Simulating taking some time"
-                  sleep 1
-                  echo '{"a":"b"}' > $out
-                '';
-              }
-              {
-                name = "skipped";
-                expected = null;
-                actual = null;
-              }
-            ];
-          };
-        };
-
         doc = {
           path = ./docs;
           deps = pp: [
@@ -236,21 +141,10 @@
         ci = {
           stages = ["test" "build" "deploy"];
           jobs = {
-            "test:flakeModule" = {
-              stage = "test";
-              script = [
-                "nix run .#nixtests:run -- --junit=junit.xml"
-              ];
-              allow_failure = true;
-              artifacts = {
-                when = "always";
-                reports.junit = "junit.xml";
-              };
-            };
             "test:lib" = {
               stage = "test";
               script = [
-                "nix run .#lib-tests -- --junit=junit.xml"
+                "nix run .#tests -- --junit=junit.xml"
               ];
               allow_failure = true;
               artifacts = {
@@ -315,10 +209,10 @@
           ntlib = import ./lib {inherit pkgs lib;};
         in {
           default = pkgs.callPackage ./package.nix {};
-          lib-tests = ntlib.mkNixtest {
+          tests = ntlib.mkNixtest {
-            modules = ntlib.autodiscover {dir = ./lib;};
+            modules = ntlib.autodiscover {dir = ./tests;};
             args = {
-              inherit pkgs;
+              inherit pkgs ntlib;
             };
           };
         };

internal/config/config.go

@@ -1,6 +1,9 @@
 package config
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/jedib0t/go-pretty/v6/text"
 	"github.com/rs/zerolog/log"
 	flag "github.com/spf13/pflag"
@@ -28,9 +31,16 @@ func Load() AppConfig {
 	flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')")
 	flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests")
 	flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring")
+	helpRequested := flag.BoolP("help", "h", false, "Show this menu")
 
 	flag.Parse()
 
+	if *helpRequested {
+		fmt.Println("Usage of nixtest:")
+		flag.PrintDefaults()
+		os.Exit(0)
+	}
+
 	if cfg.TestsFile == "" {
 		log.Panic().Msg("Tests file path (-f or --tests) is required.")
 	}

lib/default.nix

@@ -5,6 +5,8 @@
 }: let
   inherit (lib) evalModules toList;
 in rec {
+  helpers = import ./testHelpers.nix {inherit lib;};
+
   mkBinary = {
     nixtests,
     extraParams,
@@ -46,7 +48,7 @@ in rec {
 
   mkNixtestConfig = {
     modules,
-    args,
+    args ? {},
     ...
   }:
     (evalModules {

lib/lib_test.nix

@@ -1,44 +0,0 @@
-{
-  pkgs,
-  lib,
-  ...
-}: let
-  ntlib = import ./. {inherit pkgs lib;};
-in {
-  suites."Lib Tests".tests = [
-    {
-      name = "autodiscovery";
-      type = "script";
-      script = let
-        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
-          dir = ./.;
-        }));
-      in
-        # sh
-        ''
-          export PATH="${pkgs.gnugrep}/bin"
-          grep -q lib_test.nix ${actual}
-          grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual}
-        '';
-    }
-    {
-      name = "binary";
-      type = "script";
-      script = let
-        binary =
-          (ntlib.mkBinary {
-            nixtests = "stub";
-            extraParams = "--pure";
-          })
-          + "/bin/nixtests:run";
-      in
-        # sh
-        ''
-          export PATH="${pkgs.gnugrep}/bin"
-          grep -q nixtest ${binary}
-          grep -q -- "--pure" ${binary}
-          grep -q -- "--tests=stub" ${binary}
-        '';
-    }
-  ];
-}

lib/module.nix

@@ -108,11 +108,7 @@
           then val
           else
             builtins.unsafeDiscardStringContext
-            (pkgs.writeShellScript "nixtest-${config.name}" ''
+            (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath;
-              # show which line failed the test
-              set -x
-              ${val}
-            '').drvPath;
       };
     };
   };

lib/scriptHelpers.sh

@@ -0,0 +1,51 @@
+output=
+exit_code=
+
+function assert() {
+  test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; }
+}
+function assert_eq() {
+  assert "$1 -eq $2" "$3"
+}
+function assert_not_eq() {
+  assert "$1 -ne $2" "$3"
+}
+function assert_contains() {
+  echo "$1" | grep -q -- "$2" || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_not_contains() {
+  echo "$1" | grep -q -- "$2" && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_contains() {
+  grep -q -- "$2" $1 || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_not_contains() {
+  grep -q -- "$2" $1 && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+
+function tmpdir() {
+  dir=$(mktemp -d)
+  trap "rm -rf $dir" EXIT
+  echo -n "$dir"
+}
+function tmpfile() {
+  file=$(mktemp)
+  trap "rm -f $file" EXIT
+  echo -n "$file"
+}
+function run() {
+  output=$($@ 2>&1)
+  exit_code=$?
+}

lib/testHelpers.nix

@@ -0,0 +1,5 @@
+{lib, ...}: {
+  path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
+  pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
+  scriptHelpers = builtins.readFile ./scriptHelpers.sh;
+}

tests/fixtures/sample_test.nix

@@ -0,0 +1,97 @@
+{
+  lib,
+  pkgs,
+  ...
+}: {
+  skip = "skip.*d";
+  suites = {
+    "suite-one" = {
+      # required to figure out file and line, but optional
+      pos = __curPos;
+      tests = [
+        {
+          name = "test-one";
+          expected = 1;
+          actual = 1;
+        }
+        {
+          name = "fail";
+          expected = 0;
+          actual = "meow";
+        }
+        {
+          name = "snapshot-test";
+          type = "snapshot";
+          actual = "test";
+        }
+        {
+          name = "test-snapshot-drv";
+          type = "snapshot";
+          actualDrv = pkgs.runCommand "test-snapshot" {} ''
+            echo '"snapshot drv"' > $out
+          '';
+        }
+        {
+          name = "test-error-drv";
+          expected = null;
+          actualDrv = pkgs.runCommand "test-error-drv" {} ''
+            echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
+            exit 1
+          '';
+        }
+        {
+          name = "test-script";
+          type = "script";
+          script = ''
+            echo Test something here
+            # required in pure mode:
+            export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+            grep -q "test" ${builtins.toFile "test" "test"}
+          '';
+        }
+      ];
+    };
+    "other-suite".tests = [
+      {
+        name = "obj-snapshot";
+        type = "snapshot";
+        pos = __curPos;
+        actual = {hello = "world";};
+      }
+      {
+        name = "pretty-snapshot";
+        type = "snapshot";
+        format = "pretty";
+        pos = __curPos;
+        actual = {
+          example = args: {};
+          example2 = {
+            drv = pkgs.hello;
+          };
+        };
+      }
+      {
+        name = "pretty-unit";
+        format = "pretty";
+        pos = __curPos;
+        expected = pkgs.hello;
+        actual = pkgs.hello;
+      }
+      {
+        name = "test-drv";
+        pos = __curPos;
+        expected = {a = "b";};
+        actualDrv = pkgs.runCommand "test-something" {} ''
+          echo "Simulating taking some time"
+          sleep 1
+          echo '{"a":"b"}' > $out
+        '';
+      }
+      {
+        name = "skipped";
+        expected = null;
+        actual = null;
+      }
+    ];
+  };
+}

tests/lib_test.nix

@@ -0,0 +1,86 @@
+{
+  lib,
+  pkgs,
+  ntlib,
+  ...
+}: {
+  suites."Lib Tests".tests = [
+    {
+      name = "autodiscovery";
+      type = "script";
+      script = let
+        actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover {
+          dir = ./fixtures;
+        }));
+      in
+        # sh
+        ''
+          ${ntlib.helpers.path [pkgs.gnugrep]}
+          ${ntlib.helpers.scriptHelpers}
+          assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
+          assert_file_contains ${actual} "\"base\":\"/nix/store/.*-source/tests/fixtures" "should set base to fixtures dir"
+        '';
+    }
+    {
+      name = "binary";
+      type = "script";
+      script = let
+        binary =
+          (ntlib.mkBinary {
+            nixtests = "stub";
+            extraParams = "--pure";
+          })
+          + "/bin/nixtests:run";
+      in
+        # sh
+        ''
+          ${ntlib.helpers.path [pkgs.gnugrep]}
+          ${ntlib.helpers.scriptHelpers}
+          assert_file_contains ${binary} "nixtest" "should contain nixtest"
+          assert_file_contains ${binary} "--pure" "should contain --pure arg"
+          assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
+
+          run "${binary} --help"
+          assert_eq $exit_code 0 "should exit 0"
+          assert_contains "$output" "Usage of nixtest" "should show help"
+
+          run "${binary}"
+          assert_eq $exit_code 1 "should exit 1"
+          assert_contains "$output" "Tests file does not exist"
+        '';
+    }
+    {
+      name = "full run with fixtures";
+      type = "script";
+      script = let
+        binary =
+          (ntlib.mkNixtest {
+            modules = ntlib.autodiscover {dir = ./fixtures;};
+            args = {inherit pkgs;};
+          })
+          + "/bin/nixtests:run";
+      in
+        # sh
+        ''
+          ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
+          ${ntlib.helpers.scriptHelpers}
+
+          TMPDIR=$(tmpdir)
+          run "${binary} --pure --junit=$TMPDIR/junit.xml"
+          assert "$exit_code -eq 2" "should exit 2"
+          assert "-f $TMPDIR/junit.xml" "should create junit.xml"
+          assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
+
+          ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
+          run "${binary} --pure --junit=$TMPDIR/junit2.xml"
+          assert "$exit_code -eq 2" "should exit 2"
+          assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
+          assert_not_contains "$output" "executable file not found" "nix should now exist"
+          assert_contains "$output" "suite-one" "should contain suite-one"
+          assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
+          assert_contains "$output" "ERROR" "should contain an error"
+          assert_contains "$output" "SKIP" "should contain a skip"
+        '';
+    }
+  ];
+}