diff --git a/docs/examples.md b/docs/examples.md index c61c754..4e3ad41 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,6 +1,4 @@ # Example Configs -- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest) - see `flake.nix` and `tests/` - [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci) - see `tests/` + see tests/ diff --git a/docs/reference.md b/docs/reference.md deleted file mode 100644 index fe9d7cb..0000000 --- a/docs/reference.md +++ /dev/null @@ -1,53 +0,0 @@ -# Reference - -## `flakeModule` - -The `flakeModule` for [flake-parts](https://flake.parts). - -## `lib` - -### `module` - -The nix module for validation of inputs etc. -Used internally by `mkNixtestConfig`. - -### `autodiscover` - -```nix -autodiscover { - dir, - pattern ? ".*_test.nix", -} -``` - -Finds all test files in `dir` matching `pattern`. -Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg). - -### `mkNixtestConfig` - -```nix -mkNixtestConfig { - modules, - args ? {}, -} -``` - -Evaluates the test `modules`. -`args` are passed to the modules using `_module.args = args`. - -**Noteworthy attributes**: - -- `app`: nixtest wrapper -- `finalConfigJson`: derivation containing the tests json file - -### `mkNixtest` - -```nix -mkNixtest { - modules, - args ? {}, -} -``` - -Creates the nixtest wrapper, using the tests in `modules`. -Basically `(mkNixtestConfig ).app`. diff --git a/docs/usage.md b/docs/usage.md index a329017..fbcccb3 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,9 +2,6 @@ ## Flake Module -The easiest way to use Nixtest is probably using the flakeModule. -Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`: - ```nix { inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib"; @@ -37,25 +34,10 @@ Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`: ## Library -You can also use the lib directly, like this for example: +You can also integrate nixtest in your own workflow by using the lib functions directly. +Check out `flakeModule.nix` to see how it's used there. -```nix -packages.tests = ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./tests;}; - args = { - inherit pkgs ntlib; - }; -}; -``` - -This will auto-discover all test files ending with `_test.nix`. -See [reference](reference.md) for all params to `autodiscover`. - -`ntlib` can be defined like this: - -```nix -ntlib = inputs.nixtests.lib {inherit pkgs;}; -``` + ## Define Tests @@ -102,17 +84,9 @@ Examples: # to make it more reproducible and cleaner, use --pure to switch to pure # mode which will unset all env variables before running the test. That # requires you to set PATH yourself then: - # - # '' - # export PATH="${lib.makeBinPath [pkgs.gnugrep]}" - # grep -q "test" ${builtins.toFile "test" "test"} - # ''; - # - # you can also use the helpers to make it nicer to read: '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} # this adds helpers like assert etc. - assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'" + export PATH="${lib.makeBinPath [pkgs.gnugrep]}" + grep -q "test" ${builtins.toFile "test" "test"} ''; } { @@ -128,7 +102,3 @@ Examples: } ] ``` - -!!! note - - for more examples see [examples](./examples.md) diff --git a/flake.nix b/flake.nix index 72f86bd..bcc7b54 100644 --- a/flake.nix +++ b/flake.nix @@ -11,6 +11,7 @@ inputs.nix-gitlab-ci.flakeModule inputs.nix-devtools.flakeModule inputs.nix-mkdocs.flakeModule + ./lib/flakeModule.nix ]; systems = import systems; flake = {}; @@ -62,6 +63,100 @@ }; }; + nixtest = { + skip = "skip.*d"; + suites = { + "suite-one" = { + pos = __curPos; + tests = [ + { + name = "test-one"; + # required to figure out file and line, but optional + expected = 1; + actual = 1; + } + { + name = "fail"; + expected = 0; + actual = "meow"; + } + { + name = "snapshot-test"; + type = "snapshot"; + actual = "test"; + } + { + name = "test-snapshot-drv"; + type = "snapshot"; + actualDrv = pkgs.runCommand "test-snapshot" {} '' + echo '"snapshot drv"' > $out + ''; + } + { + name = "test-error-drv"; + expected = null; + actualDrv = pkgs.runCommand "test-error-drv" {} '' + echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth." + exit 1 + ''; + } + { + name = "test-script"; + type = "script"; + script = '' + echo Test something here + # required in pure mode: + export PATH="${lib.makeBinPath [pkgs.gnugrep]}" + grep -q "test" ${builtins.toFile "test" "test"} + ''; + } + ]; + }; + "other-suite".tests = [ + { + name = "obj-snapshot"; + type = "snapshot"; + pos = __curPos; + actual = {hello = "world";}; + } + { + name = "pretty-snapshot"; + type = "snapshot"; + format = "pretty"; + pos = __curPos; + actual = { + example = args: {}; + example2 = { + drv = pkgs.hello; + }; + }; + } + { + name = "pretty-unit"; + format = "pretty"; + pos = __curPos; + expected = pkgs.hello; + actual = pkgs.hello; + } + { + name = "test-drv"; + pos = __curPos; + expected = {a = "b";}; + actualDrv = pkgs.runCommand "test-something" {} '' + echo "Simulating taking some time" + sleep 1 + echo '{"a":"b"}' > $out + ''; + } + { + name = "skipped"; + expected = null; + actual = null; + } + ]; + }; + }; + doc = { path = ./docs; deps = pp: [ @@ -106,13 +201,11 @@ nav = [ {"Introduction" = "index.md";} {"Usage" = "usage.md";} - {"Reference" = "reference.md";} {"CLI" = "cli.md";} {"Example Configs" = "examples.md";} ]; markdown_extensions = [ "pymdownx.superfences" - "admonition" ]; extra.analytics = { provider = "umami"; @@ -143,10 +236,10 @@ ci = { stages = ["test" "build" "deploy"]; jobs = { - "test:lib" = { + "test" = { stage = "test"; script = [ - "nix run .#tests -- --junit=junit.xml" + "nix run .#nixtests:run -- --junit=junit.xml" ]; allow_failure = true; artifacts = { @@ -207,17 +300,7 @@ }; }; - packages = let - ntlib = import ./lib {inherit pkgs lib;}; - in { - default = pkgs.callPackage ./package.nix {}; - tests = ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./tests;}; - args = { - inherit pkgs ntlib; - }; - }; - }; + packages.default = pkgs.callPackage ./package.nix {}; }; }; diff --git a/lib/default.nix b/lib/default.nix index 2aa3683..e8b7f97 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,66 +1,62 @@ { pkgs, lib ? pkgs.lib, + self ? "", ... -}: let - inherit (lib) evalModules toList; -in rec { - helpers = import ./testHelpers.nix {inherit lib;}; - - mkBinary = { - nixtests, - extraParams, +}: { + mkTest = { + type ? "unit", + name, + description ? "", + format ? "json", + expected ? null, + actual ? null, + actualDrv ? null, + script ? null, + pos ? null, }: let - program = pkgs.callPackage ../package.nix {}; + fileRelative = lib.removePrefix ((toString self) + "/") pos.file; + actual' = + if format == "json" + then actual + else lib.generators.toPretty {} actual; + expected' = + if format == "json" + then expected + else lib.generators.toPretty {} expected; in - (pkgs.writeShellScriptBin "nixtests:run" '' - ${program}/bin/nixtest --tests=${nixtests} ${extraParams} "$@" - '') - // { - tests = nixtests; + assert lib.assertMsg (!(type == "script" && script == null)) "test ${name} has type 'script' but no script was passed"; { + inherit type name description; + actual = actual'; + expected = expected'; + # discard string context, otherwise it's being built instantly which we don't want + actualDrv = builtins.unsafeDiscardStringContext (actualDrv.drvPath or ""); + script = + if script != null + then + builtins.unsafeDiscardStringContext + (pkgs.writeShellScript "nixtest-${name}" '' + # show which line failed the test + set -x + ${script} + '').drvPath + else null; + pos = + if pos == null + then "" + else "${fileRelative}:${toString pos.line}"; }; - + mkSuite = name: tests: { + inherit name tests; + }; exportSuites = suites: let suitesList = if builtins.isList suites then suites else [suites]; - suitesMapped = builtins.toJSON suitesList; + testsMapped = builtins.toJSON suitesList; in pkgs.runCommand "tests.json" {} '' - echo '${suitesMapped}' > $out + echo '${testsMapped}' > $out ''; - - module = import ./module.nix {inherit lib pkgs;}; - - autodiscover = { - dir, - pattern ? ".*_test.nix", - }: let - files = builtins.readDir dir; - matchingFiles = builtins.filter (name: builtins.match pattern name != null) (builtins.attrNames files); - imports = map (file: /${dir}/${file}) matchingFiles; - in { - inherit imports; - # automatically set the base so test filepaths are easier to read - config.base = builtins.toString dir + "/"; - }; - - mkNixtestConfig = { - modules, - args ? {}, - ... - }: - (evalModules { - modules = - (toList modules) - ++ [ - module - { - _module.args = args; - } - ]; - }).config; - - mkNixtest = args: (mkNixtestConfig args).app; } diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 1d7e406..6068aea 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -15,14 +15,56 @@ in { nixtests-lib = import ./. {inherit pkgs self;}; in { options.nixtest = mkOption { - type = types.submodule (nixtests-lib.module); + type = types.submodule ({...}: { + options = { + skip = mkOption { + type = types.str; + default = ""; + description = "Which tests to skip (regex)"; + }; + suites = mkOption { + type = types.attrsOf (types.submodule { + options = { + tests = mkOption { + type = types.listOf types.attrs; + default = []; + }; + pos = mkOption { + type = types.nullOr types.attrs; + default = null; + }; + }; + }); + default = {}; + }; + }; + }); default = {}; }; - config.nixtest.base = toString self + "/"; - config.legacyPackages = { - "nixtests" = config.nixtest.finalConfigJson; - "nixtests:run" = config.nixtest.app; + config.legacyPackages = rec { + "nixtests" = let + suites = map (suiteName: let + suite = builtins.getAttr suiteName config.nixtest.suites; + in + nixtests-lib.mkSuite + suiteName + (map (test: + nixtests-lib.mkTest ({ + # default pos to suite's pos if given + pos = suite.pos; + } + // test)) + suite.tests)) + (builtins.attrNames config.nixtest.suites); + in + nixtests-lib.exportSuites suites; + "nixtests:run" = let + program = pkgs.callPackage ./../package.nix {}; + in + pkgs.writeShellScriptBin "nixtests:run" '' + ${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@" + ''; }; } ); diff --git a/lib/module.nix b/lib/module.nix deleted file mode 100644 index 14746e4..0000000 --- a/lib/module.nix +++ /dev/null @@ -1,188 +0,0 @@ -{ - pkgs, - lib, - ... -}: let - inherit (lib) mkOptionType mkOption types; - - nixtest-lib = import ./default.nix {inherit pkgs lib;}; - - unsetType = mkOptionType { - name = "unset"; - description = "unset"; - descriptionClass = "noun"; - check = value: true; - }; - unset = { - _type = "unset"; - }; - isUnset = lib.isType "unset"; - - filterUnset = value: - if builtins.isAttrs value && !builtins.hasAttr "_type" value - then let - filteredAttrs = builtins.mapAttrs (n: v: filterUnset v) value; - in - lib.filterAttrs (name: value: (!isUnset value)) filteredAttrs - else if builtins.isList value - then builtins.filter (elem: !isUnset elem) (map filterUnset value) - else value; - - testsSubmodule = { - config, - testsBase, - pos, - ... - }: { - options = { - pos = mkOption { - type = types.either types.attrs unsetType; - default = pos; - apply = val: - if isUnset val - then val - else let - fileRelative = lib.removePrefix testsBase val.file; - in "${fileRelative}:${toString val.line}"; - }; - type = mkOption { - type = types.enum ["unit" "snapshot" "script"]; - default = "unit"; - apply = value: - assert lib.assertMsg (value != "script" || !isUnset config.script) - "test '${config.name}' as type 'script' requires 'script' to be set"; - assert lib.assertMsg (value != "unit" || !isUnset config.expected) - "test '${config.name}' as type 'unit' requires 'expected' to be set"; - assert lib.assertMsg ( - let - actualIsUnset = isUnset config.actual; - actualDrvIsUnset = isUnset config.actualDrv; - in - (value != "unit") - || (!actualIsUnset && actualDrvIsUnset) - || (actualIsUnset && !actualDrvIsUnset) - ) - "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value; - }; - name = mkOption { - type = types.str; - }; - description = mkOption { - type = types.either types.str unsetType; - default = unset; - }; - format = mkOption { - type = types.enum ["json" "pretty"]; - default = "json"; - }; - expected = mkOption { - type = types.anything; - default = unset; - apply = val: - if isUnset val || config.format == "json" - then val - else lib.generators.toPretty {} val; - }; - actual = mkOption { - type = types.anything; - default = unset; - apply = val: - if isUnset val || config.format == "json" - then val - else lib.generators.toPretty {} val; - }; - actualDrv = mkOption { - type = types.either types.package unsetType; - default = unset; - apply = val: - # keep unset value - if isUnset val - then val - else builtins.unsafeDiscardStringContext (val.drvPath or ""); - }; - script = mkOption { - type = types.either types.str unsetType; - default = unset; - apply = val: - if isUnset val - then val - else - builtins.unsafeDiscardStringContext - (pkgs.writeShellScript "nixtest-${config.name}" val).drvPath; - }; - }; - }; - - suitesSubmodule = { - name, - config, - testsBase, - ... - }: { - options = { - name = mkOption { - type = types.str; - default = name; - }; - pos = mkOption { - type = types.either types.attrs unsetType; - default = unset; - }; - tests = mkOption { - type = types.listOf (types.submoduleWith { - modules = [testsSubmodule]; - specialArgs = { - inherit (config) pos; - inherit testsBase; - }; - }); - default = []; - }; - }; - }; - - nixtestSubmodule = {config, ...}: { - options = { - base = mkOption { - description = "Base directory of the tests, will be removed from the test file path"; - type = types.str; - default = ""; - }; - skip = mkOption { - type = types.str; - default = ""; - }; - suites = mkOption { - type = types.attrsOf (types.submoduleWith { - modules = [suitesSubmodule]; - specialArgs = { - testsBase = config.base; - }; - }); - default = {}; - apply = suites: - map ( - n: filterUnset (builtins.removeAttrs suites.${n} ["pos"]) - ) - (builtins.attrNames suites); - }; - - finalConfigJson = mkOption { - internal = true; - type = types.package; - }; - app = mkOption { - internal = true; - type = types.package; - }; - }; - config = { - finalConfigJson = nixtest-lib.exportSuites config.suites; - app = nixtest-lib.mkBinary { - nixtests = config.finalConfigJson; - extraParams = ''--skip="${config.skip}"''; - }; - }; - }; -in - nixtestSubmodule diff --git a/lib/scriptHelpers.sh b/lib/scriptHelpers.sh deleted file mode 100644 index 0b59829..0000000 --- a/lib/scriptHelpers.sh +++ /dev/null @@ -1,51 +0,0 @@ -output= -exit_code= - -function assert() { - test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; } -} -function assert_eq() { - assert "$1 -eq $2" "$3" -} -function assert_not_eq() { - assert "$1 -ne $2" "$3" -} -function assert_contains() { - echo "$1" | grep -q -- "$2" || { - echo "Assertion failed: $3. $1 does not contain $2" >&2; - exit 1; - } -} -function assert_not_contains() { - echo "$1" | grep -q -- "$2" && { - echo "Assertion failed: $3. $1 does contain $2" >&2; - exit 1; - } -} -function assert_file_contains() { - grep -q -- "$2" $1 || { - echo "Assertion failed: $3. $1 does not contain $2" >&2; - exit 1; - } -} -function assert_file_not_contains() { - grep -q -- "$2" $1 && { - echo "Assertion failed: $3. $1 does contain $2" >&2; - exit 1; - } -} - -function tmpdir() { - dir=$(mktemp -d) - trap "rm -rf $dir" EXIT - echo -n "$dir" -} -function tmpfile() { - file=$(mktemp) - trap "rm -f $file" EXIT - echo -n "$file" -} -function run() { - output=$($@ 2>&1) - exit_code=$? -} diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix deleted file mode 100644 index c86a6a8..0000000 --- a/lib/testHelpers.nix +++ /dev/null @@ -1,7 +0,0 @@ -{lib, ...}: { - path = pkgs: "export PATH=${lib.makeBinPath pkgs}"; - pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}"; - scriptHelpers = builtins.readFile ./scriptHelpers.sh; - toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any)); - toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any); -} diff --git a/tests/fixtures/sample_test.nix b/tests/fixtures/sample_test.nix deleted file mode 100644 index dcc2fb1..0000000 --- a/tests/fixtures/sample_test.nix +++ /dev/null @@ -1,97 +0,0 @@ -{ - lib, - pkgs, - ... -}: { - skip = "skip.*d"; - suites = { - "suite-one" = { - # required to figure out file and line, but optional - pos = __curPos; - tests = [ - { - name = "test-one"; - expected = 1; - actual = 1; - } - { - name = "fail"; - expected = 0; - actual = "meow"; - } - { - name = "snapshot-test"; - type = "snapshot"; - actual = "test"; - } - { - name = "test-snapshot-drv"; - type = "snapshot"; - actualDrv = pkgs.runCommand "test-snapshot" {} '' - echo '"snapshot drv"' > $out - ''; - } - { - name = "test-error-drv"; - expected = null; - actualDrv = pkgs.runCommand "test-error-drv" {} '' - echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth." - exit 1 - ''; - } - { - name = "test-script"; - type = "script"; - script = '' - echo Test something here - # required in pure mode: - export PATH="${lib.makeBinPath [pkgs.gnugrep]}" - grep -q "test" ${builtins.toFile "test" "test"} - ''; - } - ]; - }; - "other-suite".tests = [ - { - name = "obj-snapshot"; - type = "snapshot"; - pos = __curPos; - actual = {hello = "world";}; - } - { - name = "pretty-snapshot"; - type = "snapshot"; - format = "pretty"; - pos = __curPos; - actual = { - example = args: {}; - example2 = { - drv = pkgs.hello; - }; - }; - } - { - name = "pretty-unit"; - format = "pretty"; - pos = __curPos; - expected = pkgs.hello; - actual = pkgs.hello; - } - { - name = "test-drv"; - pos = __curPos; - expected = {a = "b";}; - actualDrv = pkgs.runCommand "test-something" {} '' - echo "Simulating taking some time" - sleep 1 - echo '{"a":"b"}' > $out - ''; - } - { - name = "skipped"; - expected = null; - actual = null; - } - ]; - }; -} diff --git a/tests/lib_test.nix b/tests/lib_test.nix deleted file mode 100644 index 4437cf3..0000000 --- a/tests/lib_test.nix +++ /dev/null @@ -1,90 +0,0 @@ -{ - pkgs, - ntlib, - ... -}: { - suites."Lib Tests" = { - pos = __curPos; - tests = [ - { - name = "autodiscovery"; - type = "script"; - script = let - actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover { - dir = ./fixtures; - }); - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix" - assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir" - ''; - } - { - name = "binary"; - type = "script"; - script = let - binary = - (ntlib.mkBinary { - nixtests = "stub"; - extraParams = "--pure"; - }) - + "/bin/nixtests:run"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${binary} "nixtest" "should contain nixtest" - assert_file_contains ${binary} "--pure" "should contain --pure arg" - assert_file_contains ${binary} "--tests=stub" "should contain --tests arg" - - run "${binary} --help" - assert_eq $exit_code 0 "should exit 0" - assert_contains "$output" "Usage of nixtest" "should show help" - - run "${binary}" - assert_eq $exit_code 1 "should exit 1" - assert_contains "$output" "Tests file does not exist" - ''; - } - { - name = "full run with fixtures"; - type = "script"; - script = let - binary = - (ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = ./fixtures;}; - args = {inherit pkgs;}; - }) - + "/bin/nixtests:run"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]} - ${ntlib.helpers.scriptHelpers} - - TMPDIR=$(tmpdir) - # start without nix & env binaries to expect errors - run "${binary} --pure --junit=$TMPDIR/junit.xml" - assert "$exit_code -eq 2" "should exit 2" - assert "-f $TMPDIR/junit.xml" "should create junit.xml" - assert_contains "$output" "executable file not found" "nix should not be found in pure mode" - - # now add required deps - ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]} - run "${binary} --pure --junit=$TMPDIR/junit2.xml" - assert "$exit_code -eq 2" "should exit 2" - assert "-f $TMPDIR/junit2.xml" "should create junit2.xml" - assert_not_contains "$output" "executable file not found" "nix should now exist" - assert_contains "$output" "suite-one" "should contain suite-one" - assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total" - assert_contains "$output" "ERROR" "should contain an error" - assert_contains "$output" "SKIP" "should contain a skip" - ''; - } - ]; - }; -}