diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e3ce93..24e2d78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,6 @@ All notable changes to this project will be documented in this file. See [conventional commits](https://www.conventionalcommits.org/) for commit guidelines. - - - -## [v1.4.0](https://gitlab.com/TECHNOFAB/nixtest/compare/9481833a408dde6c2a87215a8a6952a1d5de424f..v1.4.0) - 2026-10-15 -#### Features -- add new assertions test type - ([17d08b9](https://gitlab.com/TECHNOFAB/nixtest/commit/17d08b997511216cb84c4923f6f7b7d7233ec1da)) - [@TECHNOFAB](https://gitlab.com/TECHNOFAB) -#### Bug Fixes -- (**deps**) update module github.com/rs/zerolog to v1.35.1 - ([9481833](https://gitlab.com/TECHNOFAB/nixtest/commit/9481833a408dde6c2a87215a8a6952a1d5de424f)) - Renovate Bot - -- - - - ## [v1.3.0](https://gitlab.com/TECHNOFAB/nixtest/compare/dab1e2ef6f49d91174216aed8216aa98f9a071ed..v1.3.0) - 2026-04-02 #### Features - add support for running NixOS VM tests more easily - ([c64e0cc](https://gitlab.com/TECHNOFAB/nixtest/commit/c64e0cce0c5ba9d77597f20f002d1375deb8e577)) - [@TECHNOFAB](https://gitlab.com/TECHNOFAB) diff --git a/docs/reference.md b/docs/reference.md index a80dfbc..fe9d7cb 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -51,24 +51,3 @@ mkNixtest { Creates the nixtest wrapper, using the tests in `modules`. Basically `(mkNixtestConfig ).app`. - -### `asserts` / `expect` - -Assertion matchers for `type = "assertions"` tests (also available as `ntlib.helpers.expect` and `ntlib.helpers.asserts`): - -- `eq expected actual` / `equal`: checks `actual == expected`. -- `ne expected actual` / `notEqual`: checks `actual != expected`. -- `gt actual min` / `greaterThan`: checks `actual > min`. -- `gte actual min` / `greaterThanOrEqual`: checks `actual >= min`. -- `lt actual max` / `lessThan`: checks `actual < max`. -- `lte actual max` / `lessThanOrEqual`: checks `actual <= max`. -- `contains container item`: checks if a list contains an element, or a string contains a substring. -- `notContains container item`: checks a list does not contain an element, or a string does not contain a substring. -- `hasAttr attr set`: checks if an attrset has the given attribute. -- `missingAttr attr set`: checks if an attrset lacks the given attribute. -- `hasAttrs attrs set`: checks if an attrset has all attributes in the given list. -- `matches regex str`: checks if a string matches the given regex pattern. -- `isNull val` / `isNotNull`: null checks. -- `isEmpty val` / `isNotEmpty`: checks if list, string, or attrset is empty / not empty. -- `throws expr` / `fails`: asserts that evaluating `expr` throws or aborts. -- `custom cond message` / `that`: custom condition with failure explanation message. diff --git a/docs/usage.md b/docs/usage.md index 9b294b6..71b8e2f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -144,25 +144,6 @@ Examples: ''; }; } -{ - name = "assertions-test"; - type = "assertions"; - assertions = { - # Use rich matchers with automatic failure details (available as ntlib.asserts.* or ntlib.expect.*): - "equality" = ntlib.asserts.eq 200 response.status; - "greater than" = ntlib.asserts.gt config.port 1024; - "list contains" = ntlib.asserts.contains config.users "admin"; - "has attribute" = ntlib.asserts.hasAttr "nginx" config.services; - "matches regex" = ntlib.asserts.matches "^v[0-9]+" pkg.version; - "throws error" = ntlib.asserts.throws (throw "invalid config"); - - # Or simple booleans: - "is enabled" = config.enable == true; - - # Or custom failure messages: - "memory limit" = if mem > 512 then true else "memory was ${toString mem}MB (expected > 512MB)"; - }; - } ] ``` diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d4eda9e..8ccb8df 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -5,8 +5,6 @@ import ( "fmt" "reflect" "regexp" - "sort" - "strings" "sync" "time" @@ -135,8 +133,6 @@ func (r *Runner) executeTest(spec types.TestSpec) types.TestResult { r.handleUnitTest(&result, spec, actual) case types.TestTypeScript: r.handleScriptTest(&result, spec) - case types.TestTypeAssertions: - r.handleAssertionsTest(&result, spec) default: result.Status = types.StatusError result.ErrorMessage = fmt.Sprintf("Invalid test type: %s", spec.Type) @@ -197,62 +193,6 @@ func (r *Runner) handleScriptTest(result *types.TestResult, spec types.TestSpec) } } -// handleAssertionsTest processes assertions type tests -func (r *Runner) handleAssertionsTest(result *types.TestResult, spec types.TestSpec) { - if len(spec.Assertions) == 0 { - result.Status = types.StatusError - result.ErrorMessage = "No assertions defined for test" - return - } - - total := len(spec.Assertions) - failed := 0 - errored := 0 - - names := make([]string, 0, total) - for name := range spec.Assertions { - names = append(names, name) - } - sort.Strings(names) - - var sb strings.Builder - for _, name := range names { - assertRes := spec.Assertions[name] - if assertRes.Ok { - continue - } - if assertRes.Error { - errored++ - } else { - failed++ - } - - if sb.Len() > 0 { - sb.WriteString("\n\n") - } - fmt.Fprintf(&sb, "❌ \"%s\"", name) - if assertRes.Message != "" { - fmt.Fprintf(&sb, "\n %s", assertRes.Message) - } - if assertRes.Expected != "" { - fmt.Fprintf(&sb, "\n Expected: %s", assertRes.Expected) - } - if assertRes.Actual != "" { - fmt.Fprintf(&sb, "\n Actual: %s", assertRes.Actual) - } - } - - if errored > 0 { - result.Status = types.StatusError - result.ErrorMessage = sb.String() - } else if failed > 0 { - result.Status = types.StatusFailure - result.ErrorMessage = sb.String() - } else { - result.Status = types.StatusSuccess - } -} - // compareActualExpected performs the deep equality check and formats diffs func (r *Runner) compareActualExpected(result *types.TestResult, actual, expected any) { if reflect.DeepEqual(actual, expected) { diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 09e1f13..582c2c1 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -271,71 +271,6 @@ func TestRunner_executeTest(t *testing.T) { wantStatus: types.StatusFailure, wantErrMsgContains: "[exit code 1]\n[stdout]\nout on fail\n[stderr]\nerr on fail", }, - // --- Assertions Tests --- - { - name: "Assertions test all success", - spec: types.TestSpec{ - Name: "AssertionsSuccess", - Type: types.TestTypeAssertions, - Assertions: map[string]types.AssertionResult{ - "should be true": {Ok: true}, - "should match": {Ok: true}, - }, - }, - runnerConfig: Config{}, - setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {}, - wantStatus: types.StatusSuccess, - }, - { - name: "Assertions test with failure", - spec: types.TestSpec{ - Name: "AssertionsFail", - Type: types.TestTypeAssertions, - Assertions: map[string]types.AssertionResult{ - "should pass": {Ok: true}, - "should fail": { - Ok: false, - Message: "Expected values to be equal", - Expected: "10", - Actual: "5", - }, - }, - }, - runnerConfig: Config{}, - setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {}, - wantStatus: types.StatusFailure, - wantErrMsgContains: "❌ \"should fail\"\n Expected values to be equal\n Expected: 10\n Actual: 5", - }, - { - name: "Assertions test with eval error", - spec: types.TestSpec{ - Name: "AssertionsError", - Type: types.TestTypeAssertions, - Assertions: map[string]types.AssertionResult{ - "eval error": { - Ok: false, - Error: true, - Message: "Nix evaluation error (expression threw or aborted)", - }, - }, - }, - runnerConfig: Config{}, - setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {}, - wantStatus: types.StatusError, - wantErrMsgContains: "❌ \"eval error\"\n Nix evaluation error", - }, - { - name: "Assertions test with empty assertions", - spec: types.TestSpec{ - Name: "AssertionsEmpty", - Type: types.TestTypeAssertions, - Assertions: map[string]types.AssertionResult{}, - }, - runnerConfig: Config{}, - setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {}, - wantStatus: types.StatusError, - wantErrMsgContains: "No assertions defined for test", - }, } for _, tt := range tests { diff --git a/internal/types/types.go b/internal/types/types.go index e8a901b..6b39d7d 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -5,35 +5,25 @@ import "time" type TestType string const ( - TestTypeScript TestType = "script" - TestTypeUnit TestType = "unit" - TestTypeSnapshot TestType = "snapshot" - TestTypeAssertions TestType = "assertions" + TestTypeScript TestType = "script" + TestTypeUnit TestType = "unit" + TestTypeSnapshot TestType = "snapshot" ) -type AssertionResult struct { - Ok bool `json:"ok"` - Error bool `json:"error,omitempty"` - Message string `json:"message,omitempty"` - Expected string `json:"expected,omitempty"` - Actual string `json:"actual,omitempty"` -} - type SuiteSpec struct { Name string `json:"name"` Tests []TestSpec `json:"tests"` } type TestSpec struct { - Type TestType `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - Expected any `json:"expected,omitempty"` - Actual any `json:"actual,omitempty"` - ActualDrv string `json:"actualDrv,omitempty"` - Script string `json:"script,omitempty"` - Assertions map[string]AssertionResult `json:"assertions,omitempty"` - Pos string `json:"pos,omitempty"` + Type TestType `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Expected any `json:"expected,omitempty"` + Actual any `json:"actual,omitempty"` + ActualDrv string `json:"actualDrv,omitempty"` + Script string `json:"script,omitempty"` + Pos string `json:"pos,omitempty"` Suite string } diff --git a/lib/asserts.nix b/lib/asserts.nix deleted file mode 100644 index 90b70c0..0000000 --- a/lib/asserts.nix +++ /dev/null @@ -1,296 +0,0 @@ -{lib, ...}: let - inherit (lib) generators; - - # Format any value to a readable string safely - formatVal = val: - if builtins.isString val - then val - else if builtins.isBool val || builtins.isInt val || builtins.isFloat val - then builtins.toString val - else generators.toPretty {} val; - - mkFailure = { - message ? "Assertion failed", - expected ? null, - actual ? null, - }: { - ok = false; - inherit message; - expected = - if expected != null - then formatVal expected - else ""; - actual = - if actual != null - then formatVal actual - else ""; - }; -in rec { - inherit formatVal; - - equal = expected: actual: - if actual == expected - then true - else - mkFailure { - message = "Expected values to be equal"; - inherit expected actual; - }; - eq = equal; - - notEqual = expected: actual: - if actual != expected - then true - else - mkFailure { - message = "Expected values to not be equal"; - expected = "not ${formatVal expected}"; - inherit actual; - }; - ne = notEqual; - - greaterThan = actual: min: - if actual > min - then true - else - mkFailure { - message = "Expected value to be greater than ${formatVal min}"; - expected = "> ${formatVal min}"; - inherit actual; - }; - gt = greaterThan; - - greaterThanOrEqual = actual: min: - if actual >= min - then true - else - mkFailure { - message = "Expected value to be greater than or equal to ${formatVal min}"; - expected = ">= ${formatVal min}"; - inherit actual; - }; - gte = greaterThanOrEqual; - - lessThan = actual: max: - if actual < max - then true - else - mkFailure { - message = "Expected value to be less than ${formatVal max}"; - expected = "< ${formatVal max}"; - inherit actual; - }; - lt = lessThan; - - lessThanOrEqual = actual: max: - if actual <= max - then true - else - mkFailure { - message = "Expected value to be less than or equal to ${formatVal max}"; - expected = "<= ${formatVal max}"; - inherit actual; - }; - lte = lessThanOrEqual; - - contains = container: item: - if builtins.isList container - then - if builtins.elem item container - then true - else - mkFailure { - message = "Expected list to contain item"; - expected = "to contain: ${formatVal item}"; - actual = container; - } - else if builtins.isString container - then - if lib.strings.hasInfix item container - then true - else - mkFailure { - message = "Expected string to contain substring"; - expected = "to contain: ${formatVal item}"; - actual = container; - } - else - mkFailure { - message = "Expected list or string for 'contains'"; - actual = container; - }; - - notContains = container: item: - if builtins.isList container - then - if !(builtins.elem item container) - then true - else - mkFailure { - message = "Expected list to not contain item"; - expected = "not to contain: ${formatVal item}"; - actual = container; - } - else if builtins.isString container - then - if !(lib.strings.hasInfix item container) - then true - else - mkFailure { - message = "Expected string to not contain substring"; - expected = "not to contain: ${formatVal item}"; - actual = container; - } - else - mkFailure { - message = "Expected list or string for 'notContains'"; - actual = container; - }; - - hasAttr = attr: set: - if !(builtins.isAttrs set) - then - mkFailure { - message = "Expected attrset for 'hasAttr'"; - actual = set; - } - else if builtins.hasAttr attr set - then true - else - mkFailure { - message = "Expected attrset to have attribute '${attr}'"; - expected = "to have attribute: ${attr}"; - actual = "available attributes: ${formatVal (builtins.attrNames set)}"; - }; - - missingAttr = attr: set: - if !(builtins.isAttrs set) - then - mkFailure { - message = "Expected attrset for 'missingAttr'"; - actual = set; - } - else if !(builtins.hasAttr attr set) - then true - else - mkFailure { - message = "Expected attrset to not have attribute '${attr}'"; - expected = "not to have attribute: ${attr}"; - actual = "found attribute '${attr}' with value: ${formatVal set.${attr}}"; - }; - - hasAttrs = attrs: set: - if !(builtins.isAttrs set) - then - mkFailure { - message = "Expected attrset for 'hasAttrs'"; - actual = set; - } - else let - missing = builtins.filter (a: !builtins.hasAttr a set) attrs; - in - if missing == [] - then true - else - mkFailure { - message = "Expected attrset to have attributes ${formatVal attrs}"; - expected = "missing attributes: ${formatVal missing}"; - actual = "available attributes: ${formatVal (builtins.attrNames set)}"; - }; - - matches = regex: str: - if !(builtins.isString str) - then - mkFailure { - message = "Expected string for 'matches'"; - actual = str; - } - else if builtins.match regex str != null - then true - else - mkFailure { - message = "Expected string to match regex '${regex}'"; - expected = "match regex: ${regex}"; - actual = str; - }; - - isNull = val: - if val == null - then true - else - mkFailure { - message = "Expected value to be null"; - expected = "null"; - actual = val; - }; - - isNotNull = val: - if val != null - then true - else - mkFailure { - message = "Expected value to not be null"; - expected = "not null"; - actual = "null"; - }; - - isEmpty = val: let - empty = - if builtins.isList val - then val == [] - else if builtins.isAttrs val - then val == {} - else if builtins.isString val - then val == "" - else false; - in - if empty - then true - else - mkFailure { - message = "Expected value to be empty"; - expected = "empty"; - actual = val; - }; - - isNotEmpty = val: let - notEmpty = - if builtins.isList val - then val != [] - else if builtins.isAttrs val - then val != {} - else if builtins.isString val - then val != "" - else true; - in - if notEmpty - then true - else - mkFailure { - message = "Expected value to not be empty"; - expected = "not empty"; - actual = val; - }; - - throws = expr: let - res = builtins.tryEval expr; - in - if !res.success - then true - else - mkFailure { - message = "Expected expression to throw an error, but it succeeded"; - expected = "to throw"; - actual = "evaluated to: ${formatVal res.value}"; - }; - fails = throws; - - custom = cond: message: - if cond - then true - else - mkFailure { - inherit message; - }; - that = custom; -} diff --git a/lib/default.nix b/lib/default.nix index 45cd974..ec71f09 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -6,7 +6,6 @@ inherit (lib) evalModules toList; in rec { helpers = import ./testHelpers.nix {inherit lib;}; - inherit (helpers) asserts expect; mkBinary = { nixtests, diff --git a/lib/module.nix b/lib/module.nix index 70a5ecb..3028e30 100644 --- a/lib/module.nix +++ b/lib/module.nix @@ -73,9 +73,9 @@ in "${fileRelative}:${toString val.line}"; }; type = mkOption { - type = types.enum ["unit" "snapshot" "script" "assertions" "vm"]; + type = types.enum ["unit" "snapshot" "script" "vm"]; description = '' - Type of test, has to be one of "unit", "snapshot", "script", "assertions", or "vm". + Type of test, has to be one of "unit", "snapshot", "script", or "vm". ''; default = "unit"; apply = value: @@ -84,9 +84,7 @@ assert assertMsg (value == "unit" -> !isUnset config.expected) "test '${config.name}' as type 'unit' requires 'expected' to be set"; assert assertMsg (value == "unit" -> (xor (isUnset config.actual) (isUnset config.actualDrv))) - "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; - assert assertMsg (value == "assertions" -> !isUnset config.assertions) - "test '${config.name}' as type 'assertions' requires 'assertions' to be set"; value; + "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value; }; name = mkOption { type = types.str; @@ -174,16 +172,6 @@ ''; }; }; - assertions = mkUnsetOption { - type = types.attrsOf (types.oneOf [types.bool types.str types.attrs]); - description = '' - Attribute set of assertions to run for this test. - Each assertion can evaluate to: - - boolean (`true` for pass, `false` for fail) - - string (custom failure message) - - matcher result attrset (e.g. from `ntlib.asserts.*`) - ''; - }; finalConfig = mkOption { internal = true; @@ -192,75 +180,11 @@ }; config = { finalConfig = builtins.addErrorContext "[nixtest] while processing test ${config.name}" { - inherit (config) name expected actual actualDrv pos description; + inherit (config) name expected actual actualDrv; type = if config.type == "vm" then "script" else config.type; - assertions = - if config.type == "assertions" - then - assert assertMsg (!isUnset config.assertions) - "test '${config.name}' as type 'assertions' requires 'assertions' to be set"; - assert assertMsg (builtins.isAttrs config.assertions) - "test '${config.name}' assertions must be an attribute set"; - builtins.mapAttrs ( - assertName: assertionExpr: let - evalRes = builtins.tryEval assertionExpr; - in - if !evalRes.success - then { - ok = false; - error = true; - message = "Nix evaluation error (expression threw or aborted)"; - } - else let - val = evalRes.value; - in - if builtins.isBool val - then { - ok = val; - message = - if val - then "" - else "Assertion evaluated to false"; - } - else if builtins.isString val - then { - ok = false; - message = val; - } - else if builtins.isAttrs val && (val ? ok) - then { - ok = val.ok; - error = val.error or false; - message = val.message or ""; - expected = - if val ? expected - then - ( - if builtins.isString val.expected - then val.expected - else generators.toPretty {} val.expected - ) - else ""; - actual = - if val ? actual - then - ( - if builtins.isString val.actual - then val.actual - else generators.toPretty {} val.actual - ) - else ""; - } - else { - ok = false; - message = "Invalid assertion return type: expected boolean, failure message string, or matcher result"; - } - ) - config.assertions - else unset; script = if config.type == "vm" then diff --git a/lib/testHelpers.nix b/lib/testHelpers.nix index 51e0d27..c86a6a8 100644 --- a/lib/testHelpers.nix +++ b/lib/testHelpers.nix @@ -1,10 +1,7 @@ -{lib, ...}: rec { +{lib, ...}: { path = pkgs: "export PATH=${lib.makeBinPath pkgs}"; pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}"; scriptHelpers = builtins.readFile ./scriptHelpers.sh; toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any)); toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any); - - asserts = import ./asserts.nix {inherit lib;}; - expect = asserts; } diff --git a/tests/helpers_test.nix b/tests/helpers_test.nix deleted file mode 100644 index e07c04a..0000000 --- a/tests/helpers_test.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ntlib, ...}: { - suites."Helpers Tests" = { - pos = __curPos; - tests = [ - { - name = "assertion helpers"; - type = "assertions"; - assertions = with ntlib; { - "equality" = asserts.eq 42 42; - "not equal" = asserts.ne 42 100; - "greater than" = asserts.gt 10 5; - "greater than or equal" = asserts.gte 10 10; - "less than" = asserts.lt 5 10; - "less than or equal" = asserts.lte 5 5; - "list contains" = asserts.contains [1 2 3] 2; - "list not contains" = asserts.notContains [1 2 3] 4; - "string contains" = asserts.contains "hello world" "world"; - "hasAttr" = asserts.hasAttr "a" {a = 1;}; - "missingAttr" = asserts.missingAttr "b" {a = 1;}; - "hasAttrs" = asserts.hasAttrs ["a" "b"] { - a = 1; - b = 2; - }; - "matches regex" = asserts.matches "^v[0-9]+$" "v2"; - "isNull" = asserts.isNull null; - "isNotNull" = asserts.isNotNull 1; - "isEmpty" = asserts.isEmpty []; - "isNotEmpty" = asserts.isNotEmpty [1]; - "throws" = asserts.throws (throw "oops"); - "raw boolean" = true; - "expect alias" = expect.eq "foo" "foo"; - }; - } - ]; - }; -}