technofab/nixtest
1
0
Fork 0
mirror of https://gitlab.com/TECHNOFAB/nixtest.git synced 2025-12-11 17:50:11 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'feat/module-system' into 'main'

Browse source 
feat: switch to module system to evaluate suites & tests

Closes #8

See merge request TECHNOFAB/nixtest!2
This commit is contained in:
TECHNOFAB 2025-06-14 17:34:53 +02:00
commit c2a1208534
11 changed files with 594 additions and 197 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/examples.md
View file

@ -1,4 +1,6 @@
# Example Configs
- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest)
see `flake.nix` and `tests/`
- [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci)
see tests/
see `tests/`

53
docs/reference.md Normal file
View file

@ -0,0 +1,53 @@
# Reference
## `flakeModule`
The `flakeModule` for [flake-parts](https://flake.parts).
## `lib`
### `module`
The nix module for validation of inputs etc.
Used internally by `mkNixtestConfig`.
### `autodiscover`
```nix
autodiscover {
dir,
pattern ? ".*_test.nix",
}
```
Finds all test files in `dir` matching `pattern`.
Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg).
### `mkNixtestConfig`
```nix
mkNixtestConfig {
modules,
args ? {},
}
```
Evaluates the test `modules`.
`args` are passed to the modules using `_module.args = args`.
**Noteworthy attributes**:
- `app`: nixtest wrapper
- `finalConfigJson`: derivation containing the tests json file
### `mkNixtest`
```nix
mkNixtest {
modules,
args ? {},
}
```
Creates the nixtest wrapper, using the tests in `modules`.
Basically `(mkNixtestConfig <arguments>).app`.

40
docs/usage.md
View file

@ -2,6 +2,9 @@
## Flake Module
The easiest way to use Nixtest is probably using the flakeModule.
Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`:
```nix
{
inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib";
@ -34,10 +37,25 @@
## Library
You can also integrate nixtest in your own workflow by using the lib functions directly.
Check out `flakeModule.nix` to see how it's used there.
You can also use the lib directly, like this for example:
<!-- TODO: more detailed? -->
```nix
packages.tests = ntlib.mkNixtest {
modules = ntlib.autodiscover {dir = ./tests;};
args = {
inherit pkgs ntlib;
};
};
```
This will auto-discover all test files ending with `_test.nix`.
See [reference](reference.md) for all params to `autodiscover`.
`ntlib` can be defined like this:
```nix
ntlib = inputs.nixtests.lib {inherit pkgs;};
```
## Define Tests
@ -84,9 +102,17 @@ Examples:
# to make it more reproducible and cleaner, use --pure to switch to pure
# mode which will unset all env variables before running the test. That
# requires you to set PATH yourself then:
#
# ''
# export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
# grep -q "test" ${builtins.toFile "test" "test"}
# '';
#
# you can also use the helpers to make it nicer to read:
''
export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
grep -q "test" ${builtins.toFile "test" "test"}
${ntlib.helpers.path [pkgs.gnugrep]}
${ntlib.helpers.scriptHelpers} # this adds helpers like assert etc.
assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'"
'';
}
{
@ -102,3 +128,7 @@ Examples:
}
]
```
!!! note
for more examples see [examples](./examples.md)

113
flake.nix
View file

@ -11,7 +11,6 @@
inputs.nix-gitlab-ci.flakeModule
inputs.nix-devtools.flakeModule
inputs.nix-mkdocs.flakeModule
./lib/flakeModule.nix
];
systems = import systems;
flake = {};
@ -63,100 +62,6 @@
};
};
nixtest = {
skip = "skip.*d";
suites = {
"suite-one" = {
pos = __curPos;
tests = [
{
name = "test-one";
# required to figure out file and line, but optional
expected = 1;
actual = 1;
}
{
name = "fail";
expected = 0;
actual = "meow";
}
{
name = "snapshot-test";
type = "snapshot";
actual = "test";
}
{
name = "test-snapshot-drv";
type = "snapshot";
actualDrv = pkgs.runCommand "test-snapshot" {} ''
echo '"snapshot drv"' > $out
'';
}
{
name = "test-error-drv";
expected = null;
actualDrv = pkgs.runCommand "test-error-drv" {} ''
echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
exit 1
'';
}
{
name = "test-script";
type = "script";
script = ''
echo Test something here
# required in pure mode:
export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
grep -q "test" ${builtins.toFile "test" "test"}
'';
}
];
};
"other-suite".tests = [
{
name = "obj-snapshot";
type = "snapshot";
pos = __curPos;
actual = {hello = "world";};
}
{
name = "pretty-snapshot";
type = "snapshot";
format = "pretty";
pos = __curPos;
actual = {
example = args: {};
example2 = {
drv = pkgs.hello;
};
};
}
{
name = "pretty-unit";
format = "pretty";
pos = __curPos;
expected = pkgs.hello;
actual = pkgs.hello;
}
{
name = "test-drv";
pos = __curPos;
expected = {a = "b";};
actualDrv = pkgs.runCommand "test-something" {} ''
echo "Simulating taking some time"
sleep 1
echo '{"a":"b"}' > $out
'';
}
{
name = "skipped";
expected = null;
actual = null;
}
];
};
};
doc = {
path = ./docs;
deps = pp: [
@ -201,11 +106,13 @@
nav = [
{"Introduction" = "index.md";}
{"Usage" = "usage.md";}
{"Reference" = "reference.md";}
{"CLI" = "cli.md";}
{"Example Configs" = "examples.md";}
];
markdown_extensions = [
"pymdownx.superfences"
"admonition"
];
extra.analytics = {
provider = "umami";
@ -236,10 +143,10 @@
ci = {
stages = ["test" "build" "deploy"];
jobs = {
"test" = {
"test:lib" = {
stage = "test";
script = [
"nix run .#nixtests:run -- --junit=junit.xml"
"nix run .#tests -- --junit=junit.xml"
];
allow_failure = true;
artifacts = {
@ -300,7 +207,17 @@
};
};
packages.default = pkgs.callPackage ./package.nix {};
packages = let
ntlib = import ./lib {inherit pkgs lib;};
in {
default = pkgs.callPackage ./package.nix {};
tests = ntlib.mkNixtest {
modules = ntlib.autodiscover {dir = ./tests;};
args = {
inherit pkgs ntlib;
};
};
};
};
};

96
lib/default.nix
View file

@ -1,62 +1,66 @@
{
pkgs,
lib ? pkgs.lib,
self ? "",
...
}: {
mkTest = {
type ? "unit",
name,
description ? "",
format ? "json",
expected ? null,
actual ? null,
actualDrv ? null,
script ? null,
pos ? null,
}: let
inherit (lib) evalModules toList;
in rec {
helpers = import ./testHelpers.nix {inherit lib;};
mkBinary = {
nixtests,
extraParams,
}: let
fileRelative = lib.removePrefix ((toString self) + "/") pos.file;
actual' =
if format == "json"
then actual
else lib.generators.toPretty {} actual;
expected' =
if format == "json"
then expected
else lib.generators.toPretty {} expected;
program = pkgs.callPackage ../package.nix {};
in
assert lib.assertMsg (!(type == "script" && script == null)) "test ${name} has type 'script' but no script was passed"; {
inherit type name description;
actual = actual';
expected = expected';
# discard string context, otherwise it's being built instantly which we don't want
actualDrv = builtins.unsafeDiscardStringContext (actualDrv.drvPath or "");
script =
if script != null
then
builtins.unsafeDiscardStringContext
(pkgs.writeShellScript "nixtest-${name}" ''
# show which line failed the test
set -x
${script}
'').drvPath
else null;
pos =
if pos == null
then ""
else "${fileRelative}:${toString pos.line}";
(pkgs.writeShellScriptBin "nixtests:run" ''
${program}/bin/nixtest --tests=${nixtests} ${extraParams} "$@"
'')
// {
tests = nixtests;
};
mkSuite = name: tests: {
inherit name tests;
};
exportSuites = suites: let
suitesList =
if builtins.isList suites
then suites
else [suites];
testsMapped = builtins.toJSON suitesList;
suitesMapped = builtins.toJSON suitesList;
in
pkgs.runCommand "tests.json" {} ''
echo '${testsMapped}' > $out
echo '${suitesMapped}' > $out
'';
module = import ./module.nix {inherit lib pkgs;};
autodiscover = {
dir,
pattern ? ".*_test.nix",
}: let
files = builtins.readDir dir;
matchingFiles = builtins.filter (name: builtins.match pattern name != null) (builtins.attrNames files);
imports = map (file: /${dir}/${file}) matchingFiles;
in {
inherit imports;
# automatically set the base so test filepaths are easier to read
config.base = builtins.toString dir + "/";
};
mkNixtestConfig = {
modules,
args ? {},
...
}:
(evalModules {
modules =
(toList modules)
++ [
module
{
_module.args = args;
}
];
}).config;
mkNixtest = args: (mkNixtestConfig args).app;
}

52
lib/flakeModule.nix
View file

@ -15,56 +15,14 @@ in {
nixtests-lib = import ./. {inherit pkgs self;};
in {
options.nixtest = mkOption {
type = types.submodule ({...}: {
options = {
skip = mkOption {
type = types.str;
default = "";
description = "Which tests to skip (regex)";
};
suites = mkOption {
type = types.attrsOf (types.submodule {
options = {
tests = mkOption {
type = types.listOf types.attrs;
default = [];
};
pos = mkOption {
type = types.nullOr types.attrs;
default = null;
};
};
});
default = {};
};
};
});
type = types.submodule (nixtests-lib.module);
default = {};
};
config.nixtest.base = toString self + "/";
config.legacyPackages = rec {
"nixtests" = let
suites = map (suiteName: let
suite = builtins.getAttr suiteName config.nixtest.suites;
in
nixtests-lib.mkSuite
suiteName
(map (test:
nixtests-lib.mkTest ({
# default pos to suite's pos if given
pos = suite.pos;
}
// test))
suite.tests))
(builtins.attrNames config.nixtest.suites);
in
nixtests-lib.exportSuites suites;
"nixtests:run" = let
program = pkgs.callPackage ./../package.nix {};
in
pkgs.writeShellScriptBin "nixtests:run" ''
${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
'';
config.legacyPackages = {
"nixtests" = config.nixtest.finalConfigJson;
"nixtests:run" = config.nixtest.app;
};
}
);

188
lib/module.nix Normal file
View file

@ -0,0 +1,188 @@
{
pkgs,
lib,
...
}: let
inherit (lib) mkOptionType mkOption types;
nixtest-lib = import ./default.nix {inherit pkgs lib;};
unsetType = mkOptionType {
name = "unset";
description = "unset";
descriptionClass = "noun";
check = value: true;
};
unset = {
_type = "unset";
};
isUnset = lib.isType "unset";
filterUnset = value:
if builtins.isAttrs value && !builtins.hasAttr "_type" value
then let
filteredAttrs = builtins.mapAttrs (n: v: filterUnset v) value;
in
lib.filterAttrs (name: value: (!isUnset value)) filteredAttrs
else if builtins.isList value
then builtins.filter (elem: !isUnset elem) (map filterUnset value)
else value;
testsSubmodule = {
config,
testsBase,
pos,
...
}: {
options = {
pos = mkOption {
type = types.either types.attrs unsetType;
default = pos;
apply = val:
if isUnset val
then val
else let
fileRelative = lib.removePrefix testsBase val.file;
in "${fileRelative}:${toString val.line}";
};
type = mkOption {
type = types.enum ["unit" "snapshot" "script"];
default = "unit";
apply = value:
assert lib.assertMsg (value != "script" || !isUnset config.script)
"test '${config.name}' as type 'script' requires 'script' to be set";
assert lib.assertMsg (value != "unit" || !isUnset config.expected)
"test '${config.name}' as type 'unit' requires 'expected' to be set";
assert lib.assertMsg (
let
actualIsUnset = isUnset config.actual;
actualDrvIsUnset = isUnset config.actualDrv;
in
(value != "unit")
|| (!actualIsUnset && actualDrvIsUnset)
|| (actualIsUnset && !actualDrvIsUnset)
)
"test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value;
};
name = mkOption {
type = types.str;
};
description = mkOption {
type = types.either types.str unsetType;
default = unset;
};
format = mkOption {
type = types.enum ["json" "pretty"];
default = "json";
};
expected = mkOption {
type = types.anything;
default = unset;
apply = val:
if isUnset val || config.format == "json"
then val
else lib.generators.toPretty {} val;
};
actual = mkOption {
type = types.anything;
default = unset;
apply = val:
if isUnset val || config.format == "json"
then val
else lib.generators.toPretty {} val;
};
actualDrv = mkOption {
type = types.either types.package unsetType;
default = unset;
apply = val:
# keep unset value
if isUnset val
then val
else builtins.unsafeDiscardStringContext (val.drvPath or "");
};
script = mkOption {
type = types.either types.str unsetType;
default = unset;
apply = val:
if isUnset val
then val
else
builtins.unsafeDiscardStringContext
(pkgs.writeShellScript "nixtest-${config.name}" val).drvPath;
};
};
};
suitesSubmodule = {
name,
config,
testsBase,
...
}: {
options = {
name = mkOption {
type = types.str;
default = name;
};
pos = mkOption {
type = types.either types.attrs unsetType;
default = unset;
};
tests = mkOption {
type = types.listOf (types.submoduleWith {
modules = [testsSubmodule];
specialArgs = {
inherit (config) pos;
inherit testsBase;
};
});
default = [];
};
};
};
nixtestSubmodule = {config, ...}: {
options = {
base = mkOption {
description = "Base directory of the tests, will be removed from the test file path";
type = types.str;
default = "";
};
skip = mkOption {
type = types.str;
default = "";
};
suites = mkOption {
type = types.attrsOf (types.submoduleWith {
modules = [suitesSubmodule];
specialArgs = {
testsBase = config.base;
};
});
default = {};
apply = suites:
map (
n: filterUnset (builtins.removeAttrs suites.${n} ["pos"])
)
(builtins.attrNames suites);
};
finalConfigJson = mkOption {
internal = true;
type = types.package;
};
app = mkOption {
internal = true;
type = types.package;
};
};
config = {
finalConfigJson = nixtest-lib.exportSuites config.suites;
app = nixtest-lib.mkBinary {
nixtests = config.finalConfigJson;
extraParams = ''--skip="${config.skip}"'';
};
};
};
in
nixtestSubmodule

51
lib/scriptHelpers.sh Normal file
View file

@ -0,0 +1,51 @@
output=
exit_code=
function assert() {
test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; }
}
function assert_eq() {
assert "$1 -eq $2" "$3"
}
function assert_not_eq() {
assert "$1 -ne $2" "$3"
}
function assert_contains() {
echo "$1" | grep -q -- "$2" || {
echo "Assertion failed: $3. $1 does not contain $2" >&2;
exit 1;
}
}
function assert_not_contains() {
echo "$1" | grep -q -- "$2" && {
echo "Assertion failed: $3. $1 does contain $2" >&2;
exit 1;
}
}
function assert_file_contains() {
grep -q -- "$2" $1 || {
echo "Assertion failed: $3. $1 does not contain $2" >&2;
exit 1;
}
}
function assert_file_not_contains() {
grep -q -- "$2" $1 && {
echo "Assertion failed: $3. $1 does contain $2" >&2;
exit 1;
}
}
function tmpdir() {
dir=$(mktemp -d)
trap "rm -rf $dir" EXIT
echo -n "$dir"
}
function tmpfile() {
file=$(mktemp)
trap "rm -f $file" EXIT
echo -n "$file"
}
function run() {
output=$($@ 2>&1)
exit_code=$?
}

7
lib/testHelpers.nix Normal file
View file

@ -0,0 +1,7 @@
{lib, ...}: {
path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
scriptHelpers = builtins.readFile ./scriptHelpers.sh;
toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any));
toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
}

97
tests/fixtures/sample_test.nix vendored Normal file
View file

@ -0,0 +1,97 @@
{
lib,
pkgs,
...
}: {
skip = "skip.*d";
suites = {
"suite-one" = {
# required to figure out file and line, but optional
pos = __curPos;
tests = [
{
name = "test-one";
expected = 1;
actual = 1;
}
{
name = "fail";
expected = 0;
actual = "meow";
}
{
name = "snapshot-test";
type = "snapshot";
actual = "test";
}
{
name = "test-snapshot-drv";
type = "snapshot";
actualDrv = pkgs.runCommand "test-snapshot" {} ''
echo '"snapshot drv"' > $out
'';
}
{
name = "test-error-drv";
expected = null;
actualDrv = pkgs.runCommand "test-error-drv" {} ''
echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
exit 1
'';
}
{
name = "test-script";
type = "script";
script = ''
echo Test something here
# required in pure mode:
export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
grep -q "test" ${builtins.toFile "test" "test"}
'';
}
];
};
"other-suite".tests = [
{
name = "obj-snapshot";
type = "snapshot";
pos = __curPos;
actual = {hello = "world";};
}
{
name = "pretty-snapshot";
type = "snapshot";
format = "pretty";
pos = __curPos;
actual = {
example = args: {};
example2 = {
drv = pkgs.hello;
};
};
}
{
name = "pretty-unit";
format = "pretty";
pos = __curPos;
expected = pkgs.hello;
actual = pkgs.hello;
}
{
name = "test-drv";
pos = __curPos;
expected = {a = "b";};
actualDrv = pkgs.runCommand "test-something" {} ''
echo "Simulating taking some time"
sleep 1
echo '{"a":"b"}' > $out
'';
}
{
name = "skipped";
expected = null;
actual = null;
}
];
};
}

90
tests/lib_test.nix Normal file
View file

@ -0,0 +1,90 @@
{
pkgs,
ntlib,
...
}: {
suites."Lib Tests" = {
pos = __curPos;
tests = [
{
name = "autodiscovery";
type = "script";
script = let
actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover {
dir = ./fixtures;
});
in
# sh
''
${ntlib.helpers.path [pkgs.gnugrep]}
${ntlib.helpers.scriptHelpers}
assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir"
'';
}
{
name = "binary";
type = "script";
script = let
binary =
(ntlib.mkBinary {
nixtests = "stub";
extraParams = "--pure";
})
+ "/bin/nixtests:run";
in
# sh
''
${ntlib.helpers.path [pkgs.gnugrep]}
${ntlib.helpers.scriptHelpers}
assert_file_contains ${binary} "nixtest" "should contain nixtest"
assert_file_contains ${binary} "--pure" "should contain --pure arg"
assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
run "${binary} --help"
assert_eq $exit_code 0 "should exit 0"
assert_contains "$output" "Usage of nixtest" "should show help"
run "${binary}"
assert_eq $exit_code 1 "should exit 1"
assert_contains "$output" "Tests file does not exist"
'';
}
{
name = "full run with fixtures";
type = "script";
script = let
binary =
(ntlib.mkNixtest {
modules = ntlib.autodiscover {dir = ./fixtures;};
args = {inherit pkgs;};
})
+ "/bin/nixtests:run";
in
# sh
''
${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
${ntlib.helpers.scriptHelpers}
TMPDIR=$(tmpdir)
# start without nix & env binaries to expect errors
run "${binary} --pure --junit=$TMPDIR/junit.xml"
assert "$exit_code -eq 2" "should exit 2"
assert "-f $TMPDIR/junit.xml" "should create junit.xml"
assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
# now add required deps
${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
run "${binary} --pure --junit=$TMPDIR/junit2.xml"
assert "$exit_code -eq 2" "should exit 2"
assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
assert_not_contains "$output" "executable file not found" "nix should now exist"
assert_contains "$output" "suite-one" "should contain suite-one"
assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
assert_contains "$output" "ERROR" "should contain an error"
assert_contains "$output" "SKIP" "should contain a skip"
'';
}
];
};
}