Merge branch 'feat/module-system' into 'main'

feat: switch to module system to evaluate suites & tests

Closes #8

See merge request TECHNOFAB/nixtest!2

docs/examples.md

@@ -1,4 +1,6 @@
 # Example Configs
 
+- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest)
+    see `flake.nix` and `tests/`
 - [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci)
-    see tests/
+    see `tests/`

docs/reference.md

@@ -0,0 +1,53 @@
+# Reference
+
+## `flakeModule`
+
+The `flakeModule` for [flake-parts](https://flake.parts).
+
+## `lib`
+
+### `module`
+
+The nix module for validation of inputs etc.
+Used internally by `mkNixtestConfig`.
+
+### `autodiscover`
+
+```nix
+autodiscover {
+  dir,
+  pattern ? ".*_test.nix",
+}
+```
+
+Finds all test files in `dir` matching `pattern`.
+Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg).
+
+### `mkNixtestConfig`
+
+```nix
+mkNixtestConfig {
+  modules,
+  args ? {},
+}
+```
+
+Evaluates the test `modules`.
+`args` are passed to the modules using `_module.args = args`.
+
+**Noteworthy attributes**:
+
+- `app`: nixtest wrapper
+- `finalConfigJson`: derivation containing the tests json file
+
+### `mkNixtest`
+
+```nix
+mkNixtest {
+  modules,
+  args ? {},
+}
+```
+
+Creates the nixtest wrapper, using the tests in `modules`.
+Basically `(mkNixtestConfig <arguments>).app`.

docs/usage.md

@@ -2,6 +2,9 @@
 
 ## Flake Module
 
+The easiest way to use Nixtest is probably using the flakeModule.
+Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`:
+
 ```nix
 {
   inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib";
@@ -34,10 +37,25 @@
 
 ## Library
 
-You can also integrate nixtest in your own workflow by using the lib functions directly.
+You can also use the lib directly, like this for example:
-Check out `flakeModule.nix` to see how it's used there.
 
-<!-- TODO: more detailed? -->
+```nix
+packages.tests = ntlib.mkNixtest {
+  modules = ntlib.autodiscover {dir = ./tests;};
+  args = {
+    inherit pkgs ntlib;
+  };
+};
+```
+
+This will auto-discover all test files ending with `_test.nix`.
+See [reference](reference.md) for all params to `autodiscover`.
+
+`ntlib` can be defined like this:
+
+```nix
+ntlib = inputs.nixtests.lib {inherit pkgs;};
+```
 
 ## Define Tests
 
@@ -84,9 +102,17 @@ Examples:
     # to make it more reproducible and cleaner, use --pure to switch to pure
     # mode which will unset all env variables before running the test. That 
     # requires you to set PATH yourself then:
+    # 
+    # ''
+    #  export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+    #  grep -q "test" ${builtins.toFile "test" "test"}
+    # '';
+    #
+    # you can also use the helpers to make it nicer to read:
     ''
-      export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+      ${ntlib.helpers.path [pkgs.gnugrep]}
-      grep -q "test" ${builtins.toFile "test" "test"}
+      ${ntlib.helpers.scriptHelpers}  # this adds helpers like assert etc.
+      assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'"
     '';
   }
   {
@@ -102,3 +128,7 @@ Examples:
   }
 ]
 ```
+
+!!! note
+
+    for more examples see [examples](./examples.md)

flake.nix

@@ -11,7 +11,6 @@
         inputs.nix-gitlab-ci.flakeModule
         inputs.nix-devtools.flakeModule
         inputs.nix-mkdocs.flakeModule
-        ./lib/flakeModule.nix
       ];
       systems = import systems;
       flake = {};
@@ -63,100 +62,6 @@
           };
         };
 
-        nixtest = {
-          skip = "skip.*d";
-          suites = {
-            "suite-one" = {
-              pos = __curPos;
-              tests = [
-                {
-                  name = "test-one";
-                  # required to figure out file and line, but optional
-                  expected = 1;
-                  actual = 1;
-                }
-                {
-                  name = "fail";
-                  expected = 0;
-                  actual = "meow";
-                }
-                {
-                  name = "snapshot-test";
-                  type = "snapshot";
-                  actual = "test";
-                }
-                {
-                  name = "test-snapshot-drv";
-                  type = "snapshot";
-                  actualDrv = pkgs.runCommand "test-snapshot" {} ''
-                    echo '"snapshot drv"' > $out
-                  '';
-                }
-                {
-                  name = "test-error-drv";
-                  expected = null;
-                  actualDrv = pkgs.runCommand "test-error-drv" {} ''
-                    echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
-                    exit 1
-                  '';
-                }
-                {
-                  name = "test-script";
-                  type = "script";
-                  script = ''
-                    echo Test something here
-                    # required in pure mode:
-                    export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
-                    grep -q "test" ${builtins.toFile "test" "test"}
-                  '';
-                }
-              ];
-            };
-            "other-suite".tests = [
-              {
-                name = "obj-snapshot";
-                type = "snapshot";
-                pos = __curPos;
-                actual = {hello = "world";};
-              }
-              {
-                name = "pretty-snapshot";
-                type = "snapshot";
-                format = "pretty";
-                pos = __curPos;
-                actual = {
-                  example = args: {};
-                  example2 = {
-                    drv = pkgs.hello;
-                  };
-                };
-              }
-              {
-                name = "pretty-unit";
-                format = "pretty";
-                pos = __curPos;
-                expected = pkgs.hello;
-                actual = pkgs.hello;
-              }
-              {
-                name = "test-drv";
-                pos = __curPos;
-                expected = {a = "b";};
-                actualDrv = pkgs.runCommand "test-something" {} ''
-                  echo "Simulating taking some time"
-                  sleep 1
-                  echo '{"a":"b"}' > $out
-                '';
-              }
-              {
-                name = "skipped";
-                expected = null;
-                actual = null;
-              }
-            ];
-          };
-        };
-
         doc = {
           path = ./docs;
           deps = pp: [
@@ -201,11 +106,13 @@
             nav = [
               {"Introduction" = "index.md";}
               {"Usage" = "usage.md";}
+              {"Reference" = "reference.md";}
               {"CLI" = "cli.md";}
               {"Example Configs" = "examples.md";}
             ];
             markdown_extensions = [
               "pymdownx.superfences"
+              "admonition"
             ];
             extra.analytics = {
               provider = "umami";
@@ -236,10 +143,10 @@
         ci = {
           stages = ["test" "build" "deploy"];
           jobs = {
-            "test" = {
+            "test:lib" = {
               stage = "test";
               script = [
-                "nix run .#nixtests:run -- --junit=junit.xml"
+                "nix run .#tests -- --junit=junit.xml"
               ];
               allow_failure = true;
               artifacts = {
@@ -300,7 +207,17 @@
           };
         };
 
-        packages.default = pkgs.callPackage ./package.nix {};
+        packages = let
+          ntlib = import ./lib {inherit pkgs lib;};
+        in {
+          default = pkgs.callPackage ./package.nix {};
+          tests = ntlib.mkNixtest {
+            modules = ntlib.autodiscover {dir = ./tests;};
+            args = {
+              inherit pkgs ntlib;
+            };
+          };
+        };
       };
     };
 

lib/default.nix

@@ -1,62 +1,66 @@
 {
   pkgs,
   lib ? pkgs.lib,
-  self ? "",
   ...
-}: {
+}: let
-  mkTest = {
+  inherit (lib) evalModules toList;
-    type ? "unit",
+in rec {
-    name,
+  helpers = import ./testHelpers.nix {inherit lib;};
-    description ? "",
+
-    format ? "json",
+  mkBinary = {
-    expected ? null,
+    nixtests,
-    actual ? null,
+    extraParams,
-    actualDrv ? null,
-    script ? null,
-    pos ? null,
   }: let
-    fileRelative = lib.removePrefix ((toString self) + "/") pos.file;
+    program = pkgs.callPackage ../package.nix {};
-    actual' =
-      if format == "json"
-      then actual
-      else lib.generators.toPretty {} actual;
-    expected' =
-      if format == "json"
-      then expected
-      else lib.generators.toPretty {} expected;
   in
-    assert lib.assertMsg (!(type == "script" && script == null)) "test ${name} has type 'script' but no script was passed"; {
+    (pkgs.writeShellScriptBin "nixtests:run" ''
-      inherit type name description;
+      ${program}/bin/nixtest --tests=${nixtests} ${extraParams} "$@"
-      actual = actual';
+    '')
-      expected = expected';
+    // {
-      # discard string context, otherwise it's being built instantly which we don't want
+      tests = nixtests;
-      actualDrv = builtins.unsafeDiscardStringContext (actualDrv.drvPath or "");
-      script =
-        if script != null
-        then
-          builtins.unsafeDiscardStringContext
-          (pkgs.writeShellScript "nixtest-${name}" ''
-            # show which line failed the test
-            set -x
-            ${script}
-          '').drvPath
-        else null;
-      pos =
-        if pos == null
-        then ""
-        else "${fileRelative}:${toString pos.line}";
-    };
-  mkSuite = name: tests: {
-    inherit name tests;
     };
+
   exportSuites = suites: let
     suitesList =
       if builtins.isList suites
       then suites
       else [suites];
-    testsMapped = builtins.toJSON suitesList;
+    suitesMapped = builtins.toJSON suitesList;
   in
     pkgs.runCommand "tests.json" {} ''
-      echo '${testsMapped}' > $out
+      echo '${suitesMapped}' > $out
     '';
+
+  module = import ./module.nix {inherit lib pkgs;};
+
+  autodiscover = {
+    dir,
+    pattern ? ".*_test.nix",
+  }: let
+    files = builtins.readDir dir;
+    matchingFiles = builtins.filter (name: builtins.match pattern name != null) (builtins.attrNames files);
+    imports = map (file: /${dir}/${file}) matchingFiles;
+  in {
+    inherit imports;
+    # automatically set the base so test filepaths are easier to read
+    config.base = builtins.toString dir + "/";
+  };
+
+  mkNixtestConfig = {
+    modules,
+    args ? {},
+    ...
+  }:
+    (evalModules {
+      modules =
+        (toList modules)
+        ++ [
+          module
+          {
+            _module.args = args;
+          }
+        ];
+    }).config;
+
+  mkNixtest = args: (mkNixtestConfig args).app;
 }

lib/flakeModule.nix

@@ -15,56 +15,14 @@ in {
       nixtests-lib = import ./. {inherit pkgs self;};
     in {
       options.nixtest = mkOption {
-        type = types.submodule ({...}: {
+        type = types.submodule (nixtests-lib.module);
-          options = {
-            skip = mkOption {
-              type = types.str;
-              default = "";
-              description = "Which tests to skip (regex)";
-            };
-            suites = mkOption {
-              type = types.attrsOf (types.submodule {
-                options = {
-                  tests = mkOption {
-                    type = types.listOf types.attrs;
-                    default = [];
-                  };
-                  pos = mkOption {
-                    type = types.nullOr types.attrs;
-                    default = null;
-                  };
-                };
-              });
-              default = {};
-            };
-          };
-        });
         default = {};
       };
+      config.nixtest.base = toString self + "/";
 
-      config.legacyPackages = rec {
+      config.legacyPackages = {
-        "nixtests" = let
+        "nixtests" = config.nixtest.finalConfigJson;
-          suites = map (suiteName: let
+        "nixtests:run" = config.nixtest.app;
-            suite = builtins.getAttr suiteName config.nixtest.suites;
-          in
-            nixtests-lib.mkSuite
-            suiteName
-            (map (test:
-              nixtests-lib.mkTest ({
-                  # default pos to suite's pos if given
-                  pos = suite.pos;
-                }
-                // test))
-            suite.tests))
-          (builtins.attrNames config.nixtest.suites);
-        in
-          nixtests-lib.exportSuites suites;
-        "nixtests:run" = let
-          program = pkgs.callPackage ./../package.nix {};
-        in
-          pkgs.writeShellScriptBin "nixtests:run" ''
-            ${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
-          '';
       };
     }
   );

lib/module.nix

@@ -0,0 +1,188 @@
+{
+  pkgs,
+  lib,
+  ...
+}: let
+  inherit (lib) mkOptionType mkOption types;
+
+  nixtest-lib = import ./default.nix {inherit pkgs lib;};
+
+  unsetType = mkOptionType {
+    name = "unset";
+    description = "unset";
+    descriptionClass = "noun";
+    check = value: true;
+  };
+  unset = {
+    _type = "unset";
+  };
+  isUnset = lib.isType "unset";
+
+  filterUnset = value:
+    if builtins.isAttrs value && !builtins.hasAttr "_type" value
+    then let
+      filteredAttrs = builtins.mapAttrs (n: v: filterUnset v) value;
+    in
+      lib.filterAttrs (name: value: (!isUnset value)) filteredAttrs
+    else if builtins.isList value
+    then builtins.filter (elem: !isUnset elem) (map filterUnset value)
+    else value;
+
+  testsSubmodule = {
+    config,
+    testsBase,
+    pos,
+    ...
+  }: {
+    options = {
+      pos = mkOption {
+        type = types.either types.attrs unsetType;
+        default = pos;
+        apply = val:
+          if isUnset val
+          then val
+          else let
+            fileRelative = lib.removePrefix testsBase val.file;
+          in "${fileRelative}:${toString val.line}";
+      };
+      type = mkOption {
+        type = types.enum ["unit" "snapshot" "script"];
+        default = "unit";
+        apply = value:
+          assert lib.assertMsg (value != "script" || !isUnset config.script)
+          "test '${config.name}' as type 'script' requires 'script' to be set";
+          assert lib.assertMsg (value != "unit" || !isUnset config.expected)
+          "test '${config.name}' as type 'unit' requires 'expected' to be set";
+          assert lib.assertMsg (
+            let
+              actualIsUnset = isUnset config.actual;
+              actualDrvIsUnset = isUnset config.actualDrv;
+            in
+              (value != "unit")
+              || (!actualIsUnset && actualDrvIsUnset)
+              || (actualIsUnset && !actualDrvIsUnset)
+          )
+          "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value;
+      };
+      name = mkOption {
+        type = types.str;
+      };
+      description = mkOption {
+        type = types.either types.str unsetType;
+        default = unset;
+      };
+      format = mkOption {
+        type = types.enum ["json" "pretty"];
+        default = "json";
+      };
+      expected = mkOption {
+        type = types.anything;
+        default = unset;
+        apply = val:
+          if isUnset val || config.format == "json"
+          then val
+          else lib.generators.toPretty {} val;
+      };
+      actual = mkOption {
+        type = types.anything;
+        default = unset;
+        apply = val:
+          if isUnset val || config.format == "json"
+          then val
+          else lib.generators.toPretty {} val;
+      };
+      actualDrv = mkOption {
+        type = types.either types.package unsetType;
+        default = unset;
+        apply = val:
+        # keep unset value
+          if isUnset val
+          then val
+          else builtins.unsafeDiscardStringContext (val.drvPath or "");
+      };
+      script = mkOption {
+        type = types.either types.str unsetType;
+        default = unset;
+        apply = val:
+          if isUnset val
+          then val
+          else
+            builtins.unsafeDiscardStringContext
+            (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath;
+      };
+    };
+  };
+
+  suitesSubmodule = {
+    name,
+    config,
+    testsBase,
+    ...
+  }: {
+    options = {
+      name = mkOption {
+        type = types.str;
+        default = name;
+      };
+      pos = mkOption {
+        type = types.either types.attrs unsetType;
+        default = unset;
+      };
+      tests = mkOption {
+        type = types.listOf (types.submoduleWith {
+          modules = [testsSubmodule];
+          specialArgs = {
+            inherit (config) pos;
+            inherit testsBase;
+          };
+        });
+        default = [];
+      };
+    };
+  };
+
+  nixtestSubmodule = {config, ...}: {
+    options = {
+      base = mkOption {
+        description = "Base directory of the tests, will be removed from the test file path";
+        type = types.str;
+        default = "";
+      };
+      skip = mkOption {
+        type = types.str;
+        default = "";
+      };
+      suites = mkOption {
+        type = types.attrsOf (types.submoduleWith {
+          modules = [suitesSubmodule];
+          specialArgs = {
+            testsBase = config.base;
+          };
+        });
+        default = {};
+        apply = suites:
+          map (
+            n: filterUnset (builtins.removeAttrs suites.${n} ["pos"])
+          )
+          (builtins.attrNames suites);
+      };
+
+      finalConfigJson = mkOption {
+        internal = true;
+        type = types.package;
+      };
+      app = mkOption {
+        internal = true;
+        type = types.package;
+      };
+    };
+    config = {
+      finalConfigJson = nixtest-lib.exportSuites config.suites;
+      app = nixtest-lib.mkBinary {
+        nixtests = config.finalConfigJson;
+        extraParams = ''--skip="${config.skip}"'';
+      };
+    };
+  };
+in
+  nixtestSubmodule

lib/scriptHelpers.sh

@@ -0,0 +1,51 @@
+output=
+exit_code=
+
+function assert() {
+  test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; }
+}
+function assert_eq() {
+  assert "$1 -eq $2" "$3"
+}
+function assert_not_eq() {
+  assert "$1 -ne $2" "$3"
+}
+function assert_contains() {
+  echo "$1" | grep -q -- "$2" || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_not_contains() {
+  echo "$1" | grep -q -- "$2" && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_contains() {
+  grep -q -- "$2" $1 || {
+    echo "Assertion failed: $3. $1 does not contain $2" >&2;
+    exit 1;
+  }
+}
+function assert_file_not_contains() {
+  grep -q -- "$2" $1 && {
+    echo "Assertion failed: $3. $1 does contain $2" >&2;
+    exit 1;
+  }
+}
+
+function tmpdir() {
+  dir=$(mktemp -d)
+  trap "rm -rf $dir" EXIT
+  echo -n "$dir"
+}
+function tmpfile() {
+  file=$(mktemp)
+  trap "rm -f $file" EXIT
+  echo -n "$file"
+}
+function run() {
+  output=$($@ 2>&1)
+  exit_code=$?
+}

lib/testHelpers.nix

@@ -0,0 +1,7 @@
+{lib, ...}: {
+  path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
+  pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
+  scriptHelpers = builtins.readFile ./scriptHelpers.sh;
+  toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any));
+  toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
+}

tests/fixtures/sample_test.nix

@@ -0,0 +1,97 @@
+{
+  lib,
+  pkgs,
+  ...
+}: {
+  skip = "skip.*d";
+  suites = {
+    "suite-one" = {
+      # required to figure out file and line, but optional
+      pos = __curPos;
+      tests = [
+        {
+          name = "test-one";
+          expected = 1;
+          actual = 1;
+        }
+        {
+          name = "fail";
+          expected = 0;
+          actual = "meow";
+        }
+        {
+          name = "snapshot-test";
+          type = "snapshot";
+          actual = "test";
+        }
+        {
+          name = "test-snapshot-drv";
+          type = "snapshot";
+          actualDrv = pkgs.runCommand "test-snapshot" {} ''
+            echo '"snapshot drv"' > $out
+          '';
+        }
+        {
+          name = "test-error-drv";
+          expected = null;
+          actualDrv = pkgs.runCommand "test-error-drv" {} ''
+            echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
+            exit 1
+          '';
+        }
+        {
+          name = "test-script";
+          type = "script";
+          script = ''
+            echo Test something here
+            # required in pure mode:
+            export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
+            grep -q "test" ${builtins.toFile "test" "test"}
+          '';
+        }
+      ];
+    };
+    "other-suite".tests = [
+      {
+        name = "obj-snapshot";
+        type = "snapshot";
+        pos = __curPos;
+        actual = {hello = "world";};
+      }
+      {
+        name = "pretty-snapshot";
+        type = "snapshot";
+        format = "pretty";
+        pos = __curPos;
+        actual = {
+          example = args: {};
+          example2 = {
+            drv = pkgs.hello;
+          };
+        };
+      }
+      {
+        name = "pretty-unit";
+        format = "pretty";
+        pos = __curPos;
+        expected = pkgs.hello;
+        actual = pkgs.hello;
+      }
+      {
+        name = "test-drv";
+        pos = __curPos;
+        expected = {a = "b";};
+        actualDrv = pkgs.runCommand "test-something" {} ''
+          echo "Simulating taking some time"
+          sleep 1
+          echo '{"a":"b"}' > $out
+        '';
+      }
+      {
+        name = "skipped";
+        expected = null;
+        actual = null;
+      }
+    ];
+  };
+}

tests/lib_test.nix

@@ -0,0 +1,90 @@
+{
+  pkgs,
+  ntlib,
+  ...
+}: {
+  suites."Lib Tests" = {
+    pos = __curPos;
+    tests = [
+      {
+        name = "autodiscovery";
+        type = "script";
+        script = let
+          actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover {
+            dir = ./fixtures;
+          });
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep]}
+            ${ntlib.helpers.scriptHelpers}
+            assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
+            assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir"
+          '';
+      }
+      {
+        name = "binary";
+        type = "script";
+        script = let
+          binary =
+            (ntlib.mkBinary {
+              nixtests = "stub";
+              extraParams = "--pure";
+            })
+            + "/bin/nixtests:run";
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep]}
+            ${ntlib.helpers.scriptHelpers}
+            assert_file_contains ${binary} "nixtest" "should contain nixtest"
+            assert_file_contains ${binary} "--pure" "should contain --pure arg"
+            assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
+
+            run "${binary} --help"
+            assert_eq $exit_code 0 "should exit 0"
+            assert_contains "$output" "Usage of nixtest" "should show help"
+
+            run "${binary}"
+            assert_eq $exit_code 1 "should exit 1"
+            assert_contains "$output" "Tests file does not exist"
+          '';
+      }
+      {
+        name = "full run with fixtures";
+        type = "script";
+        script = let
+          binary =
+            (ntlib.mkNixtest {
+              modules = ntlib.autodiscover {dir = ./fixtures;};
+              args = {inherit pkgs;};
+            })
+            + "/bin/nixtests:run";
+        in
+          # sh
+          ''
+            ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
+            ${ntlib.helpers.scriptHelpers}
+
+            TMPDIR=$(tmpdir)
+            # start without nix & env binaries to expect errors
+            run "${binary} --pure --junit=$TMPDIR/junit.xml"
+            assert "$exit_code -eq 2" "should exit 2"
+            assert "-f $TMPDIR/junit.xml" "should create junit.xml"
+            assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
+
+            # now add required deps
+            ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
+            run "${binary} --pure --junit=$TMPDIR/junit2.xml"
+            assert "$exit_code -eq 2" "should exit 2"
+            assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
+            assert_not_contains "$output" "executable file not found" "nix should now exist"
+            assert_contains "$output" "suite-one" "should contain suite-one"
+            assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
+            assert_contains "$output" "ERROR" "should contain an error"
+            assert_contains "$output" "SKIP" "should contain a skip"
+          '';
+      }
+    ];
+  };
+}