diff --git a/docs/examples.md b/docs/examples.md index c61c754..4e3ad41 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,6 +1,4 @@ # Example Configs -- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest) - see `flake.nix` and `tests/` - [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci) - see `tests/` + see tests/ diff --git a/docs/reference.md b/docs/reference.md deleted file mode 100644 index fe9d7cb..0000000 --- a/docs/reference.md +++ /dev/null @@ -1,53 +0,0 @@ -# Reference - -## `flakeModule` - -The `flakeModule` for [flake-parts](https://flake.parts). - -## `lib` - -### `module` - -The nix module for validation of inputs etc. -Used internally by `mkNixtestConfig`. - -### `autodiscover` - -```nix -autodiscover { - dir, - pattern ? ".*_test.nix", -} -``` - -Finds all test files in `dir` matching `pattern`. -Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg). - -### `mkNixtestConfig` - -```nix -mkNixtestConfig { - modules, - args ? {}, -} -``` - -Evaluates the test `modules`. -`args` are passed to the modules using `_module.args = args`. - -**Noteworthy attributes**: - -- `app`: nixtest wrapper -- `finalConfigJson`: derivation containing the tests json file - -### `mkNixtest` - -```nix -mkNixtest { - modules, - args ? {}, -} -``` - -Creates the nixtest wrapper, using the tests in `modules`. -Basically `(mkNixtestConfig ).app`. diff --git a/docs/usage.md b/docs/usage.md index a329017..fbcccb3 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,9 +2,6 @@ ## Flake Module -The easiest way to use Nixtest is probably using the flakeModule. -Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`: - ```nix { inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib"; @@ -37,25 +34,10 @@ Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`: ## Library -You can also use the lib directly, like this for example: +You can also integrate nixtest in your own workflow by using the lib functions directly. +Check out `flakeModule.nix` to see how it's used there. -```nix -packages.tests = ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./tests;}; - args = { - inherit pkgs ntlib; - }; -}; -``` - -This will auto-discover all test files ending with `_test.nix`. -See [reference](reference.md) for all params to `autodiscover`. - -`ntlib` can be defined like this: - -```nix -ntlib = inputs.nixtests.lib {inherit pkgs;}; -``` + ## Define Tests @@ -102,17 +84,9 @@ Examples: # to make it more reproducible and cleaner, use --pure to switch to pure # mode which will unset all env variables before running the test. That # requires you to set PATH yourself then: - # - # '' - # export PATH="${lib.makeBinPath [pkgs.gnugrep]}" - # grep -q "test" ${builtins.toFile "test" "test"} - # ''; - # - # you can also use the helpers to make it nicer to read: '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} # this adds helpers like assert etc. - assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'" + export PATH="${lib.makeBinPath [pkgs.gnugrep]}" + grep -q "test" ${builtins.toFile "test" "test"} ''; } { @@ -128,7 +102,3 @@ Examples: } ] ``` - -!!! note - - for more examples see [examples](./examples.md) diff --git a/flake.nix b/flake.nix index 72f86bd..e12b9ae 100644 --- a/flake.nix +++ b/flake.nix @@ -11,6 +11,7 @@ inputs.nix-gitlab-ci.flakeModule inputs.nix-devtools.flakeModule inputs.nix-mkdocs.flakeModule + ./lib/flakeModule.nix ]; systems = import systems; flake = {}; @@ -62,6 +63,100 @@ }; }; + nixtest = { + skip = "skip.*d"; + suites = { + "suite-one" = { + pos = __curPos; + tests = [ + { + name = "test-one"; + # required to figure out file and line, but optional + expected = 1; + actual = 1; + } + { + name = "fail"; + expected = 0; + actual = "meow"; + } + { + name = "snapshot-test"; + type = "snapshot"; + actual = "test"; + } + { + name = "test-snapshot-drv"; + type = "snapshot"; + actualDrv = pkgs.runCommand "test-snapshot" {} '' + echo '"snapshot drv"' > $out + ''; + } + { + name = "test-error-drv"; + expected = null; + actualDrv = pkgs.runCommand "test-error-drv" {} '' + echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth." + exit 1 + ''; + } + { + name = "test-script"; + type = "script"; + script = '' + echo Test something here + # required in pure mode: + export PATH="${lib.makeBinPath [pkgs.gnugrep]}" + grep -q "test" ${builtins.toFile "test" "test"} + ''; + } + ]; + }; + "other-suite".tests = [ + { + name = "obj-snapshot"; + type = "snapshot"; + pos = __curPos; + actual = {hello = "world";}; + } + { + name = "pretty-snapshot"; + type = "snapshot"; + format = "pretty"; + pos = __curPos; + actual = { + example = args: {}; + example2 = { + drv = pkgs.hello; + }; + }; + } + { + name = "pretty-unit"; + format = "pretty"; + pos = __curPos; + expected = pkgs.hello; + actual = pkgs.hello; + } + { + name = "test-drv"; + pos = __curPos; + expected = {a = "b";}; + actualDrv = pkgs.runCommand "test-something" {} '' + echo "Simulating taking some time" + sleep 1 + echo '{"a":"b"}' > $out + ''; + } + { + name = "skipped"; + expected = null; + actual = null; + } + ]; + }; + }; + doc = { path = ./docs; deps = pp: [ @@ -106,13 +201,11 @@ nav = [ {"Introduction" = "index.md";} {"Usage" = "usage.md";} - {"Reference" = "reference.md";} {"CLI" = "cli.md";} {"Example Configs" = "examples.md";} ]; markdown_extensions = [ "pymdownx.superfences" - "admonition" ]; extra.analytics = { provider = "umami"; @@ -143,10 +236,21 @@ ci = { stages = ["test" "build" "deploy"]; jobs = { + "test:flakeModule" = { + stage = "test"; + script = [ + "nix run .#nixtests:run -- --junit=junit.xml" + ]; + allow_failure = true; + artifacts = { + when = "always"; + reports.junit = "junit.xml"; + }; + }; "test:lib" = { stage = "test"; script = [ - "nix run .#tests -- --junit=junit.xml" + "nix run .#lib-tests -- --junit=junit.xml" ]; allow_failure = true; artifacts = { @@ -211,10 +315,10 @@ ntlib = import ./lib {inherit pkgs lib;}; in { default = pkgs.callPackage ./package.nix {}; - tests = ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./tests;}; + lib-tests = ntlib.mkNixtest { + modules = ntlib.autodiscover {dir = ./lib;}; args = { - inherit pkgs ntlib; + inherit pkgs; }; }; }; diff --git a/internal/config/config.go b/internal/config/config.go index 9eef9bb..f0b41e6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,9 +1,6 @@ package config import ( - "fmt" - "os" - "github.com/jedib0t/go-pretty/v6/text" "github.com/rs/zerolog/log" flag "github.com/spf13/pflag" @@ -31,16 +28,9 @@ func Load() AppConfig { flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')") flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests") flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring") - helpRequested := flag.BoolP("help", "h", false, "Show this menu") flag.Parse() - if *helpRequested { - fmt.Println("Usage of nixtest:") - flag.PrintDefaults() - os.Exit(0) - } - if cfg.TestsFile == "" { log.Panic().Msg("Tests file path (-f or --tests) is required.") } diff --git a/lib/default.nix b/lib/default.nix index 2aa3683..011d50e 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -5,8 +5,6 @@ }: let inherit (lib) evalModules toList; in rec { - helpers = import ./testHelpers.nix {inherit lib;}; - mkBinary = { nixtests, extraParams, @@ -48,7 +46,7 @@ in rec { mkNixtestConfig = { modules, - args ? {}, + args, ... }: (evalModules { diff --git a/lib/lib_test.nix b/lib/lib_test.nix new file mode 100644 index 0000000..6a47538 --- /dev/null +++ b/lib/lib_test.nix @@ -0,0 +1,44 @@ +{ + pkgs, + lib, + ... +}: let + ntlib = import ./. {inherit pkgs lib;}; +in { + suites."Lib Tests".tests = [ + { + name = "autodiscovery"; + type = "script"; + script = let + actual = builtins.toFile "actual" (builtins.toJSON (ntlib.autodiscover { + dir = ./.; + })); + in + # sh + '' + export PATH="${pkgs.gnugrep}/bin" + grep -q lib_test.nix ${actual} + grep -q "\"base\":\"/nix/store/.*-source/lib/" ${actual} + ''; + } + { + name = "binary"; + type = "script"; + script = let + binary = + (ntlib.mkBinary { + nixtests = "stub"; + extraParams = "--pure"; + }) + + "/bin/nixtests:run"; + in + # sh + '' + export PATH="${pkgs.gnugrep}/bin" + grep -q nixtest ${binary} + grep -q -- "--pure" ${binary} + grep -q -- "--tests=stub" ${binary} + ''; + } + ]; +} diff --git a/lib/module.nix b/lib/module.nix index 14746e4..db5d5b9 100644 --- a/lib/module.nix +++ b/lib/module.nix @@ -108,7 +108,11 @@ then val else builtins.unsafeDiscardStringContext - (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath; + (pkgs.writeShellScript "nixtest-${config.name}" '' + # show which line failed the test + set -x + ${val} + '').drvPath; }; }; }; diff --git a/lib/scriptHelpers.sh b/lib/scriptHelpers.sh deleted file mode 100644 index 0b59829..0000000 --- a/lib/scriptHelpers.sh +++ /dev/null @@ -1,51 +0,0 @@ -output= -exit_code= - -function assert() { - test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; } -} -function assert_eq() { - assert "$1 -eq $2" "$3" -} -function assert_not_eq() { - assert "$1 -ne $2" "$3" -} -function assert_contains() { - echo "$1" | grep -q -- "$2" || { - echo "Assertion failed: $3. $1 does not contain $2" >&2; - exit 1; - } -} -function assert_not_contains() { - echo "$1" | grep -q -- "$2" && { - echo "Assertion failed: $3. $1 does contain $2" >&2; - exit 1; - } -} -function assert_file_contains() { - grep -q -- "$2" $1 || { - echo "Assertion failed: $3. $1 does not contain $2" >&2; - exit 1; - } -} -function assert_file_not_contains() { - grep -q -- "$2" $1 && { - echo "Assertion failed: $3. $1 does contain $2" >&2; - exit 1; - } -} - -function tmpdir() { - dir=$(mktemp -d) - trap "rm -rf $dir" EXIT - echo -n "$dir" -} -function tmpfile() { - file=$(mktemp) - trap "rm -f $file" EXIT - echo -n "$file" -} -function run() { - output=$($@ 2>&1) - exit_code=$? -} diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix deleted file mode 100644 index c86a6a8..0000000 --- a/lib/testHelpers.nix +++ /dev/null @@ -1,7 +0,0 @@ -{lib, ...}: { - path = pkgs: "export PATH=${lib.makeBinPath pkgs}"; - pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}"; - scriptHelpers = builtins.readFile ./scriptHelpers.sh; - toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any)); - toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any); -} diff --git a/tests/fixtures/sample_test.nix b/tests/fixtures/sample_test.nix deleted file mode 100644 index dcc2fb1..0000000 --- a/tests/fixtures/sample_test.nix +++ /dev/null @@ -1,97 +0,0 @@ -{ - lib, - pkgs, - ... -}: { - skip = "skip.*d"; - suites = { - "suite-one" = { - # required to figure out file and line, but optional - pos = __curPos; - tests = [ - { - name = "test-one"; - expected = 1; - actual = 1; - } - { - name = "fail"; - expected = 0; - actual = "meow"; - } - { - name = "snapshot-test"; - type = "snapshot"; - actual = "test"; - } - { - name = "test-snapshot-drv"; - type = "snapshot"; - actualDrv = pkgs.runCommand "test-snapshot" {} '' - echo '"snapshot drv"' > $out - ''; - } - { - name = "test-error-drv"; - expected = null; - actualDrv = pkgs.runCommand "test-error-drv" {} '' - echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth." - exit 1 - ''; - } - { - name = "test-script"; - type = "script"; - script = '' - echo Test something here - # required in pure mode: - export PATH="${lib.makeBinPath [pkgs.gnugrep]}" - grep -q "test" ${builtins.toFile "test" "test"} - ''; - } - ]; - }; - "other-suite".tests = [ - { - name = "obj-snapshot"; - type = "snapshot"; - pos = __curPos; - actual = {hello = "world";}; - } - { - name = "pretty-snapshot"; - type = "snapshot"; - format = "pretty"; - pos = __curPos; - actual = { - example = args: {}; - example2 = { - drv = pkgs.hello; - }; - }; - } - { - name = "pretty-unit"; - format = "pretty"; - pos = __curPos; - expected = pkgs.hello; - actual = pkgs.hello; - } - { - name = "test-drv"; - pos = __curPos; - expected = {a = "b";}; - actualDrv = pkgs.runCommand "test-something" {} '' - echo "Simulating taking some time" - sleep 1 - echo '{"a":"b"}' > $out - ''; - } - { - name = "skipped"; - expected = null; - actual = null; - } - ]; - }; -} diff --git a/tests/lib_test.nix b/tests/lib_test.nix deleted file mode 100644 index 4437cf3..0000000 --- a/tests/lib_test.nix +++ /dev/null @@ -1,90 +0,0 @@ -{ - pkgs, - ntlib, - ... -}: { - suites."Lib Tests" = { - pos = __curPos; - tests = [ - { - name = "autodiscovery"; - type = "script"; - script = let - actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover { - dir = ./fixtures; - }); - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix" - assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir" - ''; - } - { - name = "binary"; - type = "script"; - script = let - binary = - (ntlib.mkBinary { - nixtests = "stub"; - extraParams = "--pure"; - }) - + "/bin/nixtests:run"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${binary} "nixtest" "should contain nixtest" - assert_file_contains ${binary} "--pure" "should contain --pure arg" - assert_file_contains ${binary} "--tests=stub" "should contain --tests arg" - - run "${binary} --help" - assert_eq $exit_code 0 "should exit 0" - assert_contains "$output" "Usage of nixtest" "should show help" - - run "${binary}" - assert_eq $exit_code 1 "should exit 1" - assert_contains "$output" "Tests file does not exist" - ''; - } - { - name = "full run with fixtures"; - type = "script"; - script = let - binary = - (ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./fixtures;}; - args = {inherit pkgs;}; - }) - + "/bin/nixtests:run"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]} - ${ntlib.helpers.scriptHelpers} - - TMPDIR=$(tmpdir) - # start without nix & env binaries to expect errors - run "${binary} --pure --junit=$TMPDIR/junit.xml" - assert "$exit_code -eq 2" "should exit 2" - assert "-f $TMPDIR/junit.xml" "should create junit.xml" - assert_contains "$output" "executable file not found" "nix should not be found in pure mode" - - # now add required deps - ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]} - run "${binary} --pure --junit=$TMPDIR/junit2.xml" - assert "$exit_code -eq 2" "should exit 2" - assert "-f $TMPDIR/junit2.xml" "should create junit2.xml" - assert_not_contains "$output" "executable file not found" "nix should now exist" - assert_contains "$output" "suite-one" "should contain suite-one" - assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total" - assert_contains "$output" "ERROR" "should contain an error" - assert_contains "$output" "SKIP" "should contain a skip" - ''; - } - ]; - }; -}