Merge branch 'renovate/github.com-stretchr-testify-1.x' into 'main'

fix(deps): update module github.com/stretchr/testify to v1.12.1

See merge request TECHNOFAB/nixtest!15

CHANGELOG.md

@@ -2,6 +2,14 @@
 All notable changes to this project will be documented in this file. See [conventional commits](https://www.conventionalcommits.org/) for commit guidelines.
 
 - - -
+## [v1.4.0](https://gitlab.com/TECHNOFAB/nixtest/compare/9481833a408dde6c2a87215a8a6952a1d5de424f..v1.4.0) - 2026-10-15
+#### Features
+- add new assertions test type - ([17d08b9](https://gitlab.com/TECHNOFAB/nixtest/commit/17d08b997511216cb84c4923f6f7b7d7233ec1da)) - [@TECHNOFAB](https://gitlab.com/TECHNOFAB)
+#### Bug Fixes
+- (**deps**) update module github.com/rs/zerolog to v1.35.1 - ([9481833](https://gitlab.com/TECHNOFAB/nixtest/commit/9481833a408dde6c2a87215a8a6952a1d5de424f)) - Renovate Bot
+
+- - -
+
 ## [v1.3.0](https://gitlab.com/TECHNOFAB/nixtest/compare/dab1e2ef6f49d91174216aed8216aa98f9a071ed..v1.3.0) - 2026-04-02
 #### Features
 - add support for running NixOS VM tests more easily - ([c64e0cc](https://gitlab.com/TECHNOFAB/nixtest/commit/c64e0cce0c5ba9d77597f20f002d1375deb8e577)) - [@TECHNOFAB](https://gitlab.com/TECHNOFAB)

docs/reference.md

@@ -51,3 +51,24 @@ mkNixtest {
 
 Creates the nixtest wrapper, using the tests in `modules`.
 Basically `(mkNixtestConfig <arguments>).app`.
+
+### `asserts` / `expect`
+
+Assertion matchers for `type = "assertions"` tests (also available as `ntlib.helpers.expect` and `ntlib.helpers.asserts`):
+
+- `eq expected actual` / `equal`: checks `actual == expected`.
+- `ne expected actual` / `notEqual`: checks `actual != expected`.
+- `gt actual min` / `greaterThan`: checks `actual > min`.
+- `gte actual min` / `greaterThanOrEqual`: checks `actual >= min`.
+- `lt actual max` / `lessThan`: checks `actual < max`.
+- `lte actual max` / `lessThanOrEqual`: checks `actual <= max`.
+- `contains container item`: checks if a list contains an element, or a string contains a substring.
+- `notContains container item`: checks a list does not contain an element, or a string does not contain a substring.
+- `hasAttr attr set`: checks if an attrset has the given attribute.
+- `missingAttr attr set`: checks if an attrset lacks the given attribute.
+- `hasAttrs attrs set`: checks if an attrset has all attributes in the given list.
+- `matches regex str`: checks if a string matches the given regex pattern.
+- `isNull val` / `isNotNull`: null checks.
+- `isEmpty val` / `isNotEmpty`: checks if list, string, or attrset is empty / not empty.
+- `throws expr` / `fails`: asserts that evaluating `expr` throws or aborts.
+- `custom cond message` / `that`: custom condition with failure explanation message.

docs/usage.md

@@ -144,6 +144,25 @@ Examples:
         '';
     };
   }
+{
+    name = "assertions-test";
+    type = "assertions";
+    assertions = {
+      # Use rich matchers with automatic failure details (available as ntlib.asserts.* or ntlib.expect.*):
+      "equality" = ntlib.asserts.eq 200 response.status;
+      "greater than" = ntlib.asserts.gt config.port 1024;
+      "list contains" = ntlib.asserts.contains config.users "admin";
+      "has attribute" = ntlib.asserts.hasAttr "nginx" config.services;
+      "matches regex" = ntlib.asserts.matches "^v[0-9]+" pkg.version;
+      "throws error" = ntlib.asserts.throws (throw "invalid config");
+
+      # Or simple booleans:
+      "is enabled" = config.enable == true;
+
+      # Or custom failure messages:
+      "memory limit" = if mem > 512 then true else "memory was ${toString mem}MB (expected > 512MB)";
+    };
+  }
 ]
 ```
 

internal/runner/runner.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"reflect"
 	"regexp"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -133,6 +135,8 @@ func (r *Runner) executeTest(spec types.TestSpec) types.TestResult {
 		r.handleUnitTest(&result, spec, actual)
 	case types.TestTypeScript:
 		r.handleScriptTest(&result, spec)
+	case types.TestTypeAssertions:
+		r.handleAssertionsTest(&result, spec)
 	default:
 		result.Status = types.StatusError
 		result.ErrorMessage = fmt.Sprintf("Invalid test type: %s", spec.Type)
@@ -193,6 +197,62 @@ func (r *Runner) handleScriptTest(result *types.TestResult, spec types.TestSpec)
 	}
 }
 
+// handleAssertionsTest processes assertions type tests
+func (r *Runner) handleAssertionsTest(result *types.TestResult, spec types.TestSpec) {
+	if len(spec.Assertions) == 0 {
+		result.Status = types.StatusError
+		result.ErrorMessage = "No assertions defined for test"
+		return
+	}
+
+	total := len(spec.Assertions)
+	failed := 0
+	errored := 0
+
+	names := make([]string, 0, total)
+	for name := range spec.Assertions {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	var sb strings.Builder
+	for _, name := range names {
+		assertRes := spec.Assertions[name]
+		if assertRes.Ok {
+			continue
+		}
+		if assertRes.Error {
+			errored++
+		} else {
+			failed++
+		}
+
+		if sb.Len() > 0 {
+			sb.WriteString("\n\n")
+		}
+		fmt.Fprintf(&sb, "❌ \"%s\"", name)
+		if assertRes.Message != "" {
+			fmt.Fprintf(&sb, "\n   %s", assertRes.Message)
+		}
+		if assertRes.Expected != "" {
+			fmt.Fprintf(&sb, "\n   Expected: %s", assertRes.Expected)
+		}
+		if assertRes.Actual != "" {
+			fmt.Fprintf(&sb, "\n   Actual:   %s", assertRes.Actual)
+		}
+	}
+
+	if errored > 0 {
+		result.Status = types.StatusError
+		result.ErrorMessage = sb.String()
+	} else if failed > 0 {
+		result.Status = types.StatusFailure
+		result.ErrorMessage = sb.String()
+	} else {
+		result.Status = types.StatusSuccess
+	}
+}
+
 // compareActualExpected performs the deep equality check and formats diffs
 func (r *Runner) compareActualExpected(result *types.TestResult, actual, expected any) {
 	if reflect.DeepEqual(actual, expected) {

internal/runner/runner_test.go

@@ -271,6 +271,71 @@ func TestRunner_executeTest(t *testing.T) {
 			wantStatus:         types.StatusFailure,
 			wantErrMsgContains: "[exit code 1]\n[stdout]\nout on fail\n[stderr]\nerr on fail",
 		},
+		// --- Assertions Tests ---
+		{
+			name: "Assertions test all success",
+			spec: types.TestSpec{
+				Name: "AssertionsSuccess",
+				Type: types.TestTypeAssertions,
+				Assertions: map[string]types.AssertionResult{
+					"should be true": {Ok: true},
+					"should match":   {Ok: true},
+				},
+			},
+			runnerConfig:      Config{},
+			setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {},
+			wantStatus:        types.StatusSuccess,
+		},
+		{
+			name: "Assertions test with failure",
+			spec: types.TestSpec{
+				Name: "AssertionsFail",
+				Type: types.TestTypeAssertions,
+				Assertions: map[string]types.AssertionResult{
+					"should pass": {Ok: true},
+					"should fail": {
+						Ok:       false,
+						Message:  "Expected values to be equal",
+						Expected: "10",
+						Actual:   "5",
+					},
+				},
+			},
+			runnerConfig:       Config{},
+			setupMockServices:  func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {},
+			wantStatus:         types.StatusFailure,
+			wantErrMsgContains: "❌ \"should fail\"\n   Expected values to be equal\n   Expected: 10\n   Actual:   5",
+		},
+		{
+			name: "Assertions test with eval error",
+			spec: types.TestSpec{
+				Name: "AssertionsError",
+				Type: types.TestTypeAssertions,
+				Assertions: map[string]types.AssertionResult{
+					"eval error": {
+						Ok:      false,
+						Error:   true,
+						Message: "Nix evaluation error (expression threw or aborted)",
+					},
+				},
+			},
+			runnerConfig:       Config{},
+			setupMockServices:  func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {},
+			wantStatus:         types.StatusError,
+			wantErrMsgContains: "❌ \"eval error\"\n   Nix evaluation error",
+		},
+		{
+			name: "Assertions test with empty assertions",
+			spec: types.TestSpec{
+				Name:       "AssertionsEmpty",
+				Type:       types.TestTypeAssertions,
+				Assertions: map[string]types.AssertionResult{},
+			},
+			runnerConfig:       Config{},
+			setupMockServices:  func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {},
+			wantStatus:         types.StatusError,
+			wantErrMsgContains: "No assertions defined for test",
+		},
 	}
 
 	for _, tt := range tests {

internal/types/types.go

@@ -8,8 +8,17 @@ const (
 	TestTypeScript     TestType = "script"
 	TestTypeUnit       TestType = "unit"
 	TestTypeSnapshot   TestType = "snapshot"
+	TestTypeAssertions TestType = "assertions"
 )
 
+type AssertionResult struct {
+	Ok       bool   `json:"ok"`
+	Error    bool   `json:"error,omitempty"`
+	Message  string `json:"message,omitempty"`
+	Expected string `json:"expected,omitempty"`
+	Actual   string `json:"actual,omitempty"`
+}
+
 type SuiteSpec struct {
 	Name  string     `json:"name"`
 	Tests []TestSpec `json:"tests"`
@@ -23,6 +32,7 @@ type TestSpec struct {
 	Actual      any                        `json:"actual,omitempty"`
 	ActualDrv   string                     `json:"actualDrv,omitempty"`
 	Script      string                     `json:"script,omitempty"`
+	Assertions  map[string]AssertionResult `json:"assertions,omitempty"`
 	Pos         string                     `json:"pos,omitempty"`
 
 	Suite string

lib/asserts.nix

@@ -0,0 +1,296 @@
+{lib, ...}: let
+  inherit (lib) generators;
+
+  # Format any value to a readable string safely
+  formatVal = val:
+    if builtins.isString val
+    then val
+    else if builtins.isBool val || builtins.isInt val || builtins.isFloat val
+    then builtins.toString val
+    else generators.toPretty {} val;
+
+  mkFailure = {
+    message ? "Assertion failed",
+    expected ? null,
+    actual ? null,
+  }: {
+    ok = false;
+    inherit message;
+    expected =
+      if expected != null
+      then formatVal expected
+      else "";
+    actual =
+      if actual != null
+      then formatVal actual
+      else "";
+  };
+in rec {
+  inherit formatVal;
+
+  equal = expected: actual:
+    if actual == expected
+    then true
+    else
+      mkFailure {
+        message = "Expected values to be equal";
+        inherit expected actual;
+      };
+  eq = equal;
+
+  notEqual = expected: actual:
+    if actual != expected
+    then true
+    else
+      mkFailure {
+        message = "Expected values to not be equal";
+        expected = "not ${formatVal expected}";
+        inherit actual;
+      };
+  ne = notEqual;
+
+  greaterThan = actual: min:
+    if actual > min
+    then true
+    else
+      mkFailure {
+        message = "Expected value to be greater than ${formatVal min}";
+        expected = "> ${formatVal min}";
+        inherit actual;
+      };
+  gt = greaterThan;
+
+  greaterThanOrEqual = actual: min:
+    if actual >= min
+    then true
+    else
+      mkFailure {
+        message = "Expected value to be greater than or equal to ${formatVal min}";
+        expected = ">= ${formatVal min}";
+        inherit actual;
+      };
+  gte = greaterThanOrEqual;
+
+  lessThan = actual: max:
+    if actual < max
+    then true
+    else
+      mkFailure {
+        message = "Expected value to be less than ${formatVal max}";
+        expected = "< ${formatVal max}";
+        inherit actual;
+      };
+  lt = lessThan;
+
+  lessThanOrEqual = actual: max:
+    if actual <= max
+    then true
+    else
+      mkFailure {
+        message = "Expected value to be less than or equal to ${formatVal max}";
+        expected = "<= ${formatVal max}";
+        inherit actual;
+      };
+  lte = lessThanOrEqual;
+
+  contains = container: item:
+    if builtins.isList container
+    then
+      if builtins.elem item container
+      then true
+      else
+        mkFailure {
+          message = "Expected list to contain item";
+          expected = "to contain: ${formatVal item}";
+          actual = container;
+        }
+    else if builtins.isString container
+    then
+      if lib.strings.hasInfix item container
+      then true
+      else
+        mkFailure {
+          message = "Expected string to contain substring";
+          expected = "to contain: ${formatVal item}";
+          actual = container;
+        }
+    else
+      mkFailure {
+        message = "Expected list or string for 'contains'";
+        actual = container;
+      };
+
+  notContains = container: item:
+    if builtins.isList container
+    then
+      if !(builtins.elem item container)
+      then true
+      else
+        mkFailure {
+          message = "Expected list to not contain item";
+          expected = "not to contain: ${formatVal item}";
+          actual = container;
+        }
+    else if builtins.isString container
+    then
+      if !(lib.strings.hasInfix item container)
+      then true
+      else
+        mkFailure {
+          message = "Expected string to not contain substring";
+          expected = "not to contain: ${formatVal item}";
+          actual = container;
+        }
+    else
+      mkFailure {
+        message = "Expected list or string for 'notContains'";
+        actual = container;
+      };
+
+  hasAttr = attr: set:
+    if !(builtins.isAttrs set)
+    then
+      mkFailure {
+        message = "Expected attrset for 'hasAttr'";
+        actual = set;
+      }
+    else if builtins.hasAttr attr set
+    then true
+    else
+      mkFailure {
+        message = "Expected attrset to have attribute '${attr}'";
+        expected = "to have attribute: ${attr}";
+        actual = "available attributes: ${formatVal (builtins.attrNames set)}";
+      };
+
+  missingAttr = attr: set:
+    if !(builtins.isAttrs set)
+    then
+      mkFailure {
+        message = "Expected attrset for 'missingAttr'";
+        actual = set;
+      }
+    else if !(builtins.hasAttr attr set)
+    then true
+    else
+      mkFailure {
+        message = "Expected attrset to not have attribute '${attr}'";
+        expected = "not to have attribute: ${attr}";
+        actual = "found attribute '${attr}' with value: ${formatVal set.${attr}}";
+      };
+
+  hasAttrs = attrs: set:
+    if !(builtins.isAttrs set)
+    then
+      mkFailure {
+        message = "Expected attrset for 'hasAttrs'";
+        actual = set;
+      }
+    else let
+      missing = builtins.filter (a: !builtins.hasAttr a set) attrs;
+    in
+      if missing == []
+      then true
+      else
+        mkFailure {
+          message = "Expected attrset to have attributes ${formatVal attrs}";
+          expected = "missing attributes: ${formatVal missing}";
+          actual = "available attributes: ${formatVal (builtins.attrNames set)}";
+        };
+
+  matches = regex: str:
+    if !(builtins.isString str)
+    then
+      mkFailure {
+        message = "Expected string for 'matches'";
+        actual = str;
+      }
+    else if builtins.match regex str != null
+    then true
+    else
+      mkFailure {
+        message = "Expected string to match regex '${regex}'";
+        expected = "match regex: ${regex}";
+        actual = str;
+      };
+
+  isNull = val:
+    if val == null
+    then true
+    else
+      mkFailure {
+        message = "Expected value to be null";
+        expected = "null";
+        actual = val;
+      };
+
+  isNotNull = val:
+    if val != null
+    then true
+    else
+      mkFailure {
+        message = "Expected value to not be null";
+        expected = "not null";
+        actual = "null";
+      };
+
+  isEmpty = val: let
+    empty =
+      if builtins.isList val
+      then val == []
+      else if builtins.isAttrs val
+      then val == {}
+      else if builtins.isString val
+      then val == ""
+      else false;
+  in
+    if empty
+    then true
+    else
+      mkFailure {
+        message = "Expected value to be empty";
+        expected = "empty";
+        actual = val;
+      };
+
+  isNotEmpty = val: let
+    notEmpty =
+      if builtins.isList val
+      then val != []
+      else if builtins.isAttrs val
+      then val != {}
+      else if builtins.isString val
+      then val != ""
+      else true;
+  in
+    if notEmpty
+    then true
+    else
+      mkFailure {
+        message = "Expected value to not be empty";
+        expected = "not empty";
+        actual = val;
+      };
+
+  throws = expr: let
+    res = builtins.tryEval expr;
+  in
+    if !res.success
+    then true
+    else
+      mkFailure {
+        message = "Expected expression to throw an error, but it succeeded";
+        expected = "to throw";
+        actual = "evaluated to: ${formatVal res.value}";
+      };
+  fails = throws;
+
+  custom = cond: message:
+    if cond
+    then true
+    else
+      mkFailure {
+        inherit message;
+      };
+  that = custom;
+}

lib/default.nix

@@ -6,6 +6,7 @@
   inherit (lib) evalModules toList;
 in rec {
   helpers = import ./testHelpers.nix {inherit lib;};
+  inherit (helpers) asserts expect;
 
   mkBinary = {
     nixtests,

lib/module.nix

@@ -73,9 +73,9 @@
           in "${fileRelative}:${toString val.line}";
       };
       type = mkOption {
-        type = types.enum ["unit" "snapshot" "script" "vm"];
+        type = types.enum ["unit" "snapshot" "script" "assertions" "vm"];
         description = ''
-          Type of test, has to be one of "unit", "snapshot", "script", or "vm".
+          Type of test, has to be one of "unit", "snapshot", "script", "assertions", or "vm".
         '';
         default = "unit";
         apply = value:
@@ -84,7 +84,9 @@
           assert assertMsg (value == "unit" -> !isUnset config.expected)
           "test '${config.name}' as type 'unit' requires 'expected' to be set";
           assert assertMsg (value == "unit" -> (xor (isUnset config.actual) (isUnset config.actualDrv)))
-          "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value;
+          "test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set";
+          assert assertMsg (value == "assertions" -> !isUnset config.assertions)
+          "test '${config.name}' as type 'assertions' requires 'assertions' to be set"; value;
       };
       name = mkOption {
         type = types.str;
@@ -172,6 +174,16 @@
             '';
         };
       };
+      assertions = mkUnsetOption {
+        type = types.attrsOf (types.oneOf [types.bool types.str types.attrs]);
+        description = ''
+          Attribute set of assertions to run for this test.
+          Each assertion can evaluate to:
+          - boolean (`true` for pass, `false` for fail)
+          - string (custom failure message)
+          - matcher result attrset (e.g. from `ntlib.asserts.*`)
+        '';
+      };
 
       finalConfig = mkOption {
         internal = true;
@@ -180,11 +192,75 @@
     };
     config = {
       finalConfig = builtins.addErrorContext "[nixtest] while processing test ${config.name}" {
-        inherit (config) name expected actual actualDrv;
+        inherit (config) name expected actual actualDrv pos description;
         type =
           if config.type == "vm"
           then "script"
           else config.type;
+        assertions =
+          if config.type == "assertions"
+          then
+            assert assertMsg (!isUnset config.assertions)
+            "test '${config.name}' as type 'assertions' requires 'assertions' to be set";
+            assert assertMsg (builtins.isAttrs config.assertions)
+            "test '${config.name}' assertions must be an attribute set";
+              builtins.mapAttrs (
+                assertName: assertionExpr: let
+                  evalRes = builtins.tryEval assertionExpr;
+                in
+                  if !evalRes.success
+                  then {
+                    ok = false;
+                    error = true;
+                    message = "Nix evaluation error (expression threw or aborted)";
+                  }
+                  else let
+                    val = evalRes.value;
+                  in
+                    if builtins.isBool val
+                    then {
+                      ok = val;
+                      message =
+                        if val
+                        then ""
+                        else "Assertion evaluated to false";
+                    }
+                    else if builtins.isString val
+                    then {
+                      ok = false;
+                      message = val;
+                    }
+                    else if builtins.isAttrs val && (val ? ok)
+                    then {
+                      ok = val.ok;
+                      error = val.error or false;
+                      message = val.message or "";
+                      expected =
+                        if val ? expected
+                        then
+                          (
+                            if builtins.isString val.expected
+                            then val.expected
+                            else generators.toPretty {} val.expected
+                          )
+                        else "";
+                      actual =
+                        if val ? actual
+                        then
+                          (
+                            if builtins.isString val.actual
+                            then val.actual
+                            else generators.toPretty {} val.actual
+                          )
+                        else "";
+                    }
+                    else {
+                      ok = false;
+                      message = "Invalid assertion return type: expected boolean, failure message string, or matcher result";
+                    }
+              )
+              config.assertions
+          else unset;
         script =
           if config.type == "vm"
           then

lib/testHelpers.nix

@@ -1,7 +1,10 @@
-{lib, ...}: {
+{lib, ...}: rec {
   path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
   pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
   scriptHelpers = builtins.readFile ./scriptHelpers.sh;
   toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any));
   toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
+
+  asserts = import ./asserts.nix {inherit lib;};
+  expect = asserts;
 }

tests/helpers_test.nix

@@ -0,0 +1,36 @@
+{ntlib, ...}: {
+  suites."Helpers Tests" = {
+    pos = __curPos;
+    tests = [
+      {
+        name = "assertion helpers";
+        type = "assertions";
+        assertions = with ntlib; {
+          "equality" = asserts.eq 42 42;
+          "not equal" = asserts.ne 42 100;
+          "greater than" = asserts.gt 10 5;
+          "greater than or equal" = asserts.gte 10 10;
+          "less than" = asserts.lt 5 10;
+          "less than or equal" = asserts.lte 5 5;
+          "list contains" = asserts.contains [1 2 3] 2;
+          "list not contains" = asserts.notContains [1 2 3] 4;
+          "string contains" = asserts.contains "hello world" "world";
+          "hasAttr" = asserts.hasAttr "a" {a = 1;};
+          "missingAttr" = asserts.missingAttr "b" {a = 1;};
+          "hasAttrs" = asserts.hasAttrs ["a" "b"] {
+            a = 1;
+            b = 2;
+          };
+          "matches regex" = asserts.matches "^v[0-9]+$" "v2";
+          "isNull" = asserts.isNull null;
+          "isNotNull" = asserts.isNotNull 1;
+          "isEmpty" = asserts.isEmpty [];
+          "isNotEmpty" = asserts.isNotEmpty [1];
+          "throws" = asserts.throws (throw "oops");
+          "raw boolean" = true;
+          "expect alias" = expect.eq "foo" "foo";
+        };
+      }
+    ];
+  };
+}