diff --git a/docs/examples.md b/docs/examples.md index 4e3ad41..c61c754 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,4 +1,6 @@ # Example Configs +- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest) + see `flake.nix` and `tests/` - [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci) - see tests/ + see `tests/` diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..fe9d7cb --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,53 @@ +# Reference + +## `flakeModule` + +The `flakeModule` for [flake-parts](https://flake.parts). + +## `lib` + +### `module` + +The nix module for validation of inputs etc. +Used internally by `mkNixtestConfig`. + +### `autodiscover` + +```nix +autodiscover { + dir, + pattern ? ".*_test.nix", +} +``` + +Finds all test files in `dir` matching `pattern`. +Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg). + +### `mkNixtestConfig` + +```nix +mkNixtestConfig { + modules, + args ? {}, +} +``` + +Evaluates the test `modules`. +`args` are passed to the modules using `_module.args = args`. + +**Noteworthy attributes**: + +- `app`: nixtest wrapper +- `finalConfigJson`: derivation containing the tests json file + +### `mkNixtest` + +```nix +mkNixtest { + modules, + args ? {}, +} +``` + +Creates the nixtest wrapper, using the tests in `modules`. +Basically `(mkNixtestConfig ).app`. diff --git a/docs/usage.md b/docs/usage.md index fbcccb3..a329017 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,6 +2,9 @@ ## Flake Module +The easiest way to use Nixtest is probably using the flakeModule. +Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`: + ```nix { inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib"; @@ -34,10 +37,25 @@ ## Library -You can also integrate nixtest in your own workflow by using the lib functions directly. -Check out `flakeModule.nix` to see how it's used there. +You can also use the lib directly, like this for example: - +```nix +packages.tests = ntlib.mkNixtest { + modules = ntlib.autodiscover {dir = ./tests;}; + args = { + inherit pkgs ntlib; + }; +}; +``` + +This will auto-discover all test files ending with `_test.nix`. +See [reference](reference.md) for all params to `autodiscover`. + +`ntlib` can be defined like this: + +```nix +ntlib = inputs.nixtests.lib {inherit pkgs;}; +``` ## Define Tests @@ -84,9 +102,17 @@ Examples: # to make it more reproducible and cleaner, use --pure to switch to pure # mode which will unset all env variables before running the test. That # requires you to set PATH yourself then: + # + # '' + # export PATH="${lib.makeBinPath [pkgs.gnugrep]}" + # grep -q "test" ${builtins.toFile "test" "test"} + # ''; + # + # you can also use the helpers to make it nicer to read: '' - export PATH="${lib.makeBinPath [pkgs.gnugrep]}" - grep -q "test" ${builtins.toFile "test" "test"} + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} # this adds helpers like assert etc. + assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'" ''; } { @@ -102,3 +128,7 @@ Examples: } ] ``` + +!!! note + + for more examples see [examples](./examples.md) diff --git a/flake.nix b/flake.nix index e12b9ae..72f86bd 100644 --- a/flake.nix +++ b/flake.nix @@ -11,7 +11,6 @@ inputs.nix-gitlab-ci.flakeModule inputs.nix-devtools.flakeModule inputs.nix-mkdocs.flakeModule - ./lib/flakeModule.nix ]; systems = import systems; flake = {}; @@ -63,100 +62,6 @@ }; }; - nixtest = { - skip = "skip.*d"; - suites = { - "suite-one" = { - pos = __curPos; - tests = [ - { - name = "test-one"; - # required to figure out file and line, but optional - expected = 1; - actual = 1; - } - { - name = "fail"; - expected = 0; - actual = "meow"; - } - { - name = "snapshot-test"; - type = "snapshot"; - actual = "test"; - } - { - name = "test-snapshot-drv"; - type = "snapshot"; - actualDrv = pkgs.runCommand "test-snapshot" {} '' - echo '"snapshot drv"' > $out - ''; - } - { - name = "test-error-drv"; - expected = null; - actualDrv = pkgs.runCommand "test-error-drv" {} '' - echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth." - exit 1 - ''; - } - { - name = "test-script"; - type = "script"; - script = '' - echo Test something here - # required in pure mode: - export PATH="${lib.makeBinPath [pkgs.gnugrep]}" - grep -q "test" ${builtins.toFile "test" "test"} - ''; - } - ]; - }; - "other-suite".tests = [ - { - name = "obj-snapshot"; - type = "snapshot"; - pos = __curPos; - actual = {hello = "world";}; - } - { - name = "pretty-snapshot"; - type = "snapshot"; - format = "pretty"; - pos = __curPos; - actual = { - example = args: {}; - example2 = { - drv = pkgs.hello; - }; - }; - } - { - name = "pretty-unit"; - format = "pretty"; - pos = __curPos; - expected = pkgs.hello; - actual = pkgs.hello; - } - { - name = "test-drv"; - pos = __curPos; - expected = {a = "b";}; - actualDrv = pkgs.runCommand "test-something" {} '' - echo "Simulating taking some time" - sleep 1 - echo '{"a":"b"}' > $out - ''; - } - { - name = "skipped"; - expected = null; - actual = null; - } - ]; - }; - }; - doc = { path = ./docs; deps = pp: [ @@ -201,11 +106,13 @@ nav = [ {"Introduction" = "index.md";} {"Usage" = "usage.md";} + {"Reference" = "reference.md";} {"CLI" = "cli.md";} {"Example Configs" = "examples.md";} ]; markdown_extensions = [ "pymdownx.superfences" + "admonition" ]; extra.analytics = { provider = "umami"; @@ -236,21 +143,10 @@ ci = { stages = ["test" "build" "deploy"]; jobs = { - "test:flakeModule" = { - stage = "test"; - script = [ - "nix run .#nixtests:run -- --junit=junit.xml" - ]; - allow_failure = true; - artifacts = { - when = "always"; - reports.junit = "junit.xml"; - }; - }; "test:lib" = { stage = "test"; script = [ - "nix run .#lib-tests -- --junit=junit.xml" + "nix run .#tests -- --junit=junit.xml" ]; allow_failure = true; artifacts = { @@ -315,10 +211,10 @@ ntlib = import ./lib {inherit pkgs lib;}; in { default = pkgs.callPackage ./package.nix {}; - lib-tests = ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./lib;}; + tests = ntlib.mkNixtest { + modules = ntlib.autodiscover {dir = ./tests;}; args = { - inherit pkgs; + inherit pkgs ntlib; }; }; }; diff --git a/internal/config/config.go b/internal/config/config.go index f0b41e6..9eef9bb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,9 @@ package config import ( + "fmt" + "os" + "github.com/jedib0t/go-pretty/v6/text" "github.com/rs/zerolog/log" flag "github.com/spf13/pflag" @@ -28,9 +31,16 @@ func Load() AppConfig { flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')") flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests") flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring") + helpRequested := flag.BoolP("help", "h", false, "Show this menu") flag.Parse() + if *helpRequested { + fmt.Println("Usage of nixtest:") + flag.PrintDefaults() + os.Exit(0) + } + if cfg.TestsFile == "" { log.Panic().Msg("Tests file path (-f or --tests) is required.") } diff --git a/lib/default.nix b/lib/default.nix index 011d50e..2aa3683 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -5,6 +5,8 @@ }: let inherit (lib) evalModules toList; in rec { + helpers = import ./testHelpers.nix {inherit lib;}; + mkBinary = { nixtests, extraParams, @@ -46,7 +48,7 @@ in rec { mkNixtestConfig = { modules, - args, + args ? {}, ... }: (evalModules { diff --git a/lib/lib_test.nix b/lib/lib_test.nix deleted file mode 100644 index 6a47538..0000000 --- a/lib/lib_test.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ - pkgs, - lib, - ... -}: let - ntlib = import ./. {inherit pkgs lib;}; -in { - suites."Lib Tests".tests = [ - { - name = "autodiscovery"; - type = "script"; - script = let - actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover { - dir = ./.; - })); - in - # sh - '' - export PATH="${pkgs.gnugrep}/bin" - grep -q lib_test.nix ${actual} - grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual} - ''; - } - { - name = "binary"; - type = "script"; - script = let - binary = - (ntlib.mkBinary { - nixtests = "stub"; - extraParams = "--pure"; - }) - + "/bin/nixtests:run"; - in - # sh - '' - export PATH="${pkgs.gnugrep}/bin" - grep -q nixtest ${binary} - grep -q -- "--pure" ${binary} - grep -q -- "--tests=stub" ${binary} - ''; - } - ]; -} diff --git a/lib/module.nix b/lib/module.nix index db5d5b9..14746e4 100644 --- a/lib/module.nix +++ b/lib/module.nix @@ -108,11 +108,7 @@ then val else builtins.unsafeDiscardStringContext - (pkgs.writeShellScript "nixtest-${config.name}" '' - # show which line failed the test - set -x - ${val} - '').drvPath; + (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath; }; }; }; diff --git a/lib/scriptHelpers.sh b/lib/scriptHelpers.sh new file mode 100644 index 0000000..0b59829 --- /dev/null +++ b/lib/scriptHelpers.sh @@ -0,0 +1,51 @@ +output= +exit_code= + +function assert() { + test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; } +} +function assert_eq() { + assert "$1 -eq $2" "$3" +} +function assert_not_eq() { + assert "$1 -ne $2" "$3" +} +function assert_contains() { + echo "$1" | grep -q -- "$2" || { + echo "Assertion failed: $3. $1 does not contain $2" >&2; + exit 1; + } +} +function assert_not_contains() { + echo "$1" | grep -q -- "$2" && { + echo "Assertion failed: $3. $1 does contain $2" >&2; + exit 1; + } +} +function assert_file_contains() { + grep -q -- "$2" $1 || { + echo "Assertion failed: $3. $1 does not contain $2" >&2; + exit 1; + } +} +function assert_file_not_contains() { + grep -q -- "$2" $1 && { + echo "Assertion failed: $3. $1 does contain $2" >&2; + exit 1; + } +} + +function tmpdir() { + dir=$(mktemp -d) + trap "rm -rf $dir" EXIT + echo -n "$dir" +} +function tmpfile() { + file=$(mktemp) + trap "rm -f $file" EXIT + echo -n "$file" +} +function run() { + output=$($@ 2>&1) + exit_code=$? +} diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix new file mode 100644 index 0000000..c86a6a8 --- /dev/null +++ b/lib/testHelpers.nix @@ -0,0 +1,7 @@ +{lib, ...}: { + path = pkgs: "export PATH=${lib.makeBinPath pkgs}"; + pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}"; + scriptHelpers = builtins.readFile ./scriptHelpers.sh; + toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any)); + toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any); +} diff --git a/tests/fixtures/sample_test.nix b/tests/fixtures/sample_test.nix new file mode 100644 index 0000000..dcc2fb1 --- /dev/null +++ b/tests/fixtures/sample_test.nix @@ -0,0 +1,97 @@ +{ + lib, + pkgs, + ... +}: { + skip = "skip.*d"; + suites = { + "suite-one" = { + # required to figure out file and line, but optional + pos = __curPos; + tests = [ + { + name = "test-one"; + expected = 1; + actual = 1; + } + { + name = "fail"; + expected = 0; + actual = "meow"; + } + { + name = "snapshot-test"; + type = "snapshot"; + actual = "test"; + } + { + name = "test-snapshot-drv"; + type = "snapshot"; + actualDrv = pkgs.runCommand "test-snapshot" {} '' + echo '"snapshot drv"' > $out + ''; + } + { + name = "test-error-drv"; + expected = null; + actualDrv = pkgs.runCommand "test-error-drv" {} '' + echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth." + exit 1 + ''; + } + { + name = "test-script"; + type = "script"; + script = '' + echo Test something here + # required in pure mode: + export PATH="${lib.makeBinPath [pkgs.gnugrep]}" + grep -q "test" ${builtins.toFile "test" "test"} + ''; + } + ]; + }; + "other-suite".tests = [ + { + name = "obj-snapshot"; + type = "snapshot"; + pos = __curPos; + actual = {hello = "world";}; + } + { + name = "pretty-snapshot"; + type = "snapshot"; + format = "pretty"; + pos = __curPos; + actual = { + example = args: {}; + example2 = { + drv = pkgs.hello; + }; + }; + } + { + name = "pretty-unit"; + format = "pretty"; + pos = __curPos; + expected = pkgs.hello; + actual = pkgs.hello; + } + { + name = "test-drv"; + pos = __curPos; + expected = {a = "b";}; + actualDrv = pkgs.runCommand "test-something" {} '' + echo "Simulating taking some time" + sleep 1 + echo '{"a":"b"}' > $out + ''; + } + { + name = "skipped"; + expected = null; + actual = null; + } + ]; + }; +} diff --git a/tests/lib_test.nix b/tests/lib_test.nix new file mode 100644 index 0000000..4437cf3 --- /dev/null +++ b/tests/lib_test.nix @@ -0,0 +1,90 @@ +{ + pkgs, + ntlib, + ... +}: { + suites."Lib Tests" = { + pos = __curPos; + tests = [ + { + name = "autodiscovery"; + type = "script"; + script = let + actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover { + dir = ./fixtures; + }); + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix" + assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir" + ''; + } + { + name = "binary"; + type = "script"; + script = let + binary = + (ntlib.mkBinary { + nixtests = "stub"; + extraParams = "--pure"; + }) + + "/bin/nixtests:run"; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${binary} "nixtest" "should contain nixtest" + assert_file_contains ${binary} "--pure" "should contain --pure arg" + assert_file_contains ${binary} "--tests=stub" "should contain --tests arg" + + run "${binary} --help" + assert_eq $exit_code 0 "should exit 0" + assert_contains "$output" "Usage of nixtest" "should show help" + + run "${binary}" + assert_eq $exit_code 1 "should exit 1" + assert_contains "$output" "Tests file does not exist" + ''; + } + { + name = "full run with fixtures"; + type = "script"; + script = let + binary = + (ntlib.mkNixtest { + modules = ntlib.autodiscover {dir = ./fixtures;}; + args = {inherit pkgs;}; + }) + + "/bin/nixtests:run"; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]} + ${ntlib.helpers.scriptHelpers} + + TMPDIR=$(tmpdir) + # start without nix & env binaries to expect errors + run "${binary} --pure --junit=$TMPDIR/junit.xml" + assert "$exit_code -eq 2" "should exit 2" + assert "-f $TMPDIR/junit.xml" "should create junit.xml" + assert_contains "$output" "executable file not found" "nix should not be found in pure mode" + + # now add required deps + ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]} + run "${binary} --pure --junit=$TMPDIR/junit2.xml" + assert "$exit_code -eq 2" "should exit 2" + assert "-f $TMPDIR/junit2.xml" "should create junit2.xml" + assert_not_contains "$output" "executable file not found" "nix should now exist" + assert_contains "$output" "suite-one" "should contain suite-one" + assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total" + assert_contains "$output" "ERROR" "should contain an error" + assert_contains "$output" "SKIP" "should contain a skip" + ''; + } + ]; + }; +}