feat: general improvements and add junit "error" and "skipped" support

cmd/nixtest/display.go

@@ -8,20 +8,23 @@ import (
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/jedib0t/go-pretty/v6/text"
 	"github.com/rs/zerolog/log"
+	"github.com/sergi/go-diff/diffmatchpatch"
 )
 
 func printErrors(results Results) {
 	for _, suiteResults := range results {
 		for _, result := range suiteResults {
-			if result.Success {
+			if result.Status == StatusSuccess || result.Status == StatusSkipped {
 				continue
 			}
-			fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Name))
+			fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Spec.Name))
 			var message string
-			if result.Error.Diff != "" {
+			if result.Status == StatusFailure {
-				message = fmt.Sprintf("Diff:\n%s", result.Error.Diff)
+				dmp := diffmatchpatch.New()
+				diffs := dmp.DiffMain(result.Expected, result.Actual, false)
+				message = fmt.Sprintf("Diff:\n%s", dmp.DiffPrettyText(diffs))
 			} else {
-				message = result.Error.Message
+				message = result.ErrorMessage
 			}
 			for line := range strings.Lines(message) {
 				fmt.Printf("%s %s", text.FgRed.Sprint("|"), line)
@@ -47,7 +50,7 @@ func printSummary(results Results, successCount int, totalCount int) {
 		suiteSuccess := 0
 
 		for _, res := range suiteResults {
-			if res.Success {
+			if res.Status == StatusSuccess || res.Status == StatusSkipped {
 				suiteSuccess++
 			}
 		}
@@ -60,15 +63,19 @@ func printSummary(results Results, successCount int, totalCount int) {
 		})
 		for _, res := range suiteResults {
 			symbol := "❌"
-			if res.Success {
+			if res.Status == StatusSuccess {
 				symbol = "✅"
+			} else if res.Status == StatusError {
+				symbol = "error"
+			} else if res.Status == StatusSkipped {
+				symbol = "skipped"
 			}
 
 			t.AppendRow([]any{
-				res.Name,
+				res.Spec.Name,
 				fmt.Sprintf("%s", res.Duration),
 				symbol,
-				res.Pos,
+				res.Spec.Pos,
 			})
 		}
 		t.AppendSeparator()

cmd/nixtest/junit.go

@@ -13,6 +13,8 @@ type JUnitReport struct {
 	Name     string           `xml:"name,attr"`
 	Tests    int              `xml:"tests,attr"`
 	Failures int              `xml:"failures,attr"`
+	Errors   int              `xml:"errors,attr"`
+	Skipped  int              `xml:"skipped,attr"`
 	Time     string           `xml:"time,attr"` // in seconds
 	Suites   []JUnitTestSuite `xml:"testsuite"`
 }
@@ -22,6 +24,8 @@ type JUnitTestSuite struct {
 	Name      string      `xml:"name,attr"`
 	Tests     int         `xml:"tests,attr"`
 	Failures  int         `xml:"failures,attr"`
+	Errors    int         `xml:"errors,attr"`
+	Skipped   int         `xml:"skipped,attr"`
 	Time      string      `xml:"time,attr"` // in seconds
 	TestCases []JUnitCase `xml:"testcase"`
 }
@@ -34,6 +38,7 @@ type JUnitCase struct {
 	Line      string  `xml:"line,attr,omitempty"`
 	Failure   *string `xml:"failure,omitempty"`
 	Error     *string `xml:"error,omitempty"`
+	Skipped   *string `xml:"skipped,omitempty"`
 }
 
 func GenerateJUnitReport(name string, results Results) (string, error) {
@@ -41,6 +46,8 @@ func GenerateJUnitReport(name string, results Results) (string, error) {
 		Name:     name,
 		Tests:    0,
 		Failures: 0,
+		Errors:   0,
+		Skipped:  0,
 		Suites:   []JUnitTestSuite{},
 	}
 
@@ -51,6 +58,8 @@ func GenerateJUnitReport(name string, results Results) (string, error) {
 			Name:      suiteName,
 			Tests:     len(suiteResults),
 			Failures:  0,
+			Errors:    0,
+			Skipped:   0,
 			TestCases: []JUnitCase{},
 		}
 
@@ -62,21 +71,35 @@ func GenerateJUnitReport(name string, results Results) (string, error) {
 			suiteDuration += result.Duration
 
 			testCase := JUnitCase{
-				Name:      result.Name,
+				Name:      result.Spec.Name,
 				Classname: suiteName, // Use suite name as classname
 				Time:      durationSeconds,
 			}
 
-			if result.Pos != "" {
+			if result.Spec.Pos != "" {
-				pos := strings.Split(result.Pos, ":")
+				pos := strings.Split(result.Spec.Pos, ":")
 				testCase.File = pos[0]
 				testCase.Line = pos[1]
 			}
 
-			if !result.Success {
+			if result.Status == StatusFailure {
 				suite.Failures++
 				report.Failures++
-				testCase.Failure = &result.Error.Message
+				message := fmt.Sprintf(
+					"Expected:\n%s\nGot:\n%s",
+					PrefixLines(result.Expected),
+					PrefixLines(result.Actual),
+				)
+				testCase.Failure = &message
+			} else if result.Status == StatusError {
+				suite.Errors++
+				report.Errors++
+				testCase.Error = &result.ErrorMessage
+			} else if result.Status == StatusSkipped {
+				suite.Skipped++
+				report.Skipped++
+				skipped := ""
+				testCase.Skipped = &skipped
 			}
 
 			suite.TestCases = append(suite.TestCases, testCase)

cmd/nixtest/main.go

@@ -9,13 +9,13 @@ import (
 	"os/exec"
 	"path"
 	"reflect"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
-	"github.com/sergi/go-diff/diffmatchpatch"
 	flag "github.com/spf13/pflag"
 )
 
@@ -36,13 +36,22 @@ type TestSpec struct {
 	Suite string
 }
 
+type TestStatus int
+
+const (
+	StatusSuccess TestStatus = iota
+	StatusFailure
+	StatusError
+	StatusSkipped
+)
+
 type TestResult struct {
-	Name     string
+	Spec         TestSpec
-	Success  bool
+	Status       TestStatus
-	Error    TestResultError
+	Duration     time.Duration
-	Duration time.Duration
+	ErrorMessage string
-	Pos      string
+	Expected     string
-	Suite    string
+	Actual       string
 }
 
 type Results map[string][]TestResult
@@ -80,13 +89,6 @@ func buildAndParse(derivation string) (any, error) {
 	return result, nil
 }
 
-type TestResultError struct {
-	Message  string
-	Expected string
-	Actual   string
-	Diff     string
-}
-
 func PrefixLines(input string) string {
 	lines := strings.Split(input, "\n")
 	for i := range lines {
@@ -95,14 +97,29 @@ func PrefixLines(input string) string {
 	return strings.Join(lines, "\n")
 }
 
+func shouldSkip(input string, pattern string) bool {
+	if pattern == "" {
+		return false
+	}
+
+	regex, err := regexp.Compile(pattern)
+	if err != nil {
+		log.Panic().Err(err).Msg("failed to compile skip regex")
+	}
+
+	return regex.MatchString(input)
+}
+
 func runTest(spec TestSpec) TestResult {
 	startTime := time.Now()
 	result := TestResult{
-		Name:    spec.Name,
+		Spec:   spec,
-		Pos:     spec.Pos,
+		Status: StatusSuccess,
-		Suite:   spec.Suite,
+	}
-		Success: false,
+
-		Error:   TestResultError{},
+	if shouldSkip(spec.Name, *skipPattern) {
+		result.Status = StatusSkipped
+		return result
 	}
 
 	var actual any
@@ -112,7 +129,8 @@ func runTest(spec TestSpec) TestResult {
 		var err error
 		actual, err = buildAndParse(spec.ActualDrv)
 		if err != nil {
-			result.Error.Message = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error())
+			result.Status = StatusError
+			result.ErrorMessage = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error())
 			goto end
 		}
 	} else {
@@ -129,14 +147,16 @@ func runTest(spec TestSpec) TestResult {
 		}
 
 		if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
-			result.Error.Message = "No Snapshot exists yet"
+			result.Status = StatusError
+			result.ErrorMessage = "No Snapshot exists yet"
 			goto end
 		}
 
 		var err error
 		expected, err = ParseFile[any](filePath)
 		if err != nil {
-			result.Error.Message = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error())
+			result.Status = StatusError
+			result.ErrorMessage = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error())
 			goto end
 		}
 	} else if spec.Type == "unit" {
@@ -146,28 +166,24 @@ func runTest(spec TestSpec) TestResult {
 	}
 
 	if reflect.DeepEqual(actual, expected) {
-		result.Success = true
+		result.Status = StatusSuccess
 	} else {
-		dmp := diffmatchpatch.New()
 		text1, err := json.MarshalIndent(expected, "", "  ")
 		if err != nil {
-			result.Error.Message = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error())
+			result.Status = StatusError
+			result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error())
 			goto end
 		}
 		text2, err := json.MarshalIndent(actual, "", "  ")
 		if err != nil {
-			result.Error.Message = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error())
+			result.Status = StatusError
+			result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error())
 			goto end
 		}
-		diffs := dmp.DiffMain(string(text1), string(text2), false)
+
-		result.Error.Expected = string(text1)
+		result.Status = StatusFailure
-		result.Error.Actual = string(text2)
+		result.Expected = string(text1)
-		result.Error.Diff = dmp.DiffPrettyText(diffs)
+		result.Actual = string(text2)
-		result.Error.Message = fmt.Sprintf(
-			"Expected:\n%s\nGot:\n%s",
-			PrefixLines(string(text1)),
-			PrefixLines(string(text2)),
-		)
 	}
 
 end:
@@ -211,7 +227,8 @@ var (
 	junitPath *string = flag.String(
 		"junit", "", "Path to generate JUNIT report to, leave empty to disable",
 	)
-	updateSnapshots *bool = flag.Bool("update-snapshots", false, "Update all snapshots")
+	updateSnapshots *bool   = flag.Bool("update-snapshots", false, "Update all snapshots")
+	skipPattern     *string = flag.String("skip", "", "Regular expression to skip (e.g., 'test-.*|.*-b')")
 )
 
 func main() {
@@ -269,8 +286,8 @@ func main() {
 	successCount := 0
 
 	for r := range resultsChan {
-		results[r.Suite] = append(results[r.Suite], r)
+		results[r.Spec.Suite] = append(results[r.Spec.Suite], r)
-		if r.Success {
+		if r.Status == StatusSuccess || r.Status == StatusSkipped {
 			successCount++
 		}
 	}

flake.nix

@@ -41,54 +41,71 @@
           };
         };
 
-        testSuites = {
+        nixtest = {
-          "suite-one" = [
+          skip = "skip.*d";
-            {
+          suites = {
-              name = "test-one";
+            "suite-one" = [
-              # required to figure out file and line, but optional
+              {
-              pos = __curPos;
+                name = "test-one";
-              expected = 1;
+                # required to figure out file and line, but optional
-              actual = 1;
+                pos = __curPos;
-            }
+                expected = 1;
-            {
+                actual = 1;
-              name = "fail";
+              }
-              pos = __curPos;
+              {
-              expected = 0;
+                name = "fail";
-              actual = "meow";
+                pos = __curPos;
-            }
+                expected = 0;
-            {
+                actual = "meow";
-              name = "snapshot-test";
+              }
-              type = "snapshot";
+              {
-              pos = __curPos;
+                name = "snapshot-test";
-              actual = "test";
+                type = "snapshot";
-            }
+                pos = __curPos;
-            {
+                actual = "test";
-              name = "test-snapshot-drv";
+              }
-              type = "snapshot";
+              {
-              pos = __curPos;
+                name = "test-snapshot-drv";
-              actualDrv = pkgs.runCommand "test-snapshot" {} ''
+                type = "snapshot";
-                echo '"snapshot drv"' > $out
+                pos = __curPos;
-              '';
+                actualDrv = pkgs.runCommand "test-snapshot" {} ''
-            }
+                  echo '"snapshot drv"' > $out
-          ];
+                '';
-          "other-suite" = [
+              }
-            {
+              {
-              name = "obj-snapshot";
+                name = "test-error-drv";
-              type = "snapshot";
+                pos = __curPos;
-              pos = __curPos;
+                expected = null;
-              actual = {hello = "world";};
+                actualDrv = pkgs.runCommand "test-error-drv" {} ''
-            }
+                  exit 1
-            {
+                '';
-              name = "test-drv";
+              }
-              pos = __curPos;
+            ];
-              expected = {a = "b";};
+            "other-suite" = [
-              actualDrv = pkgs.runCommand "test-something" {} ''
+              {
-                echo "Simulating taking some time"
+                name = "obj-snapshot";
-                sleep 1
+                type = "snapshot";
-                echo '{"a":"b"}' > $out
+                pos = __curPos;
-              '';
+                actual = {hello = "world";};
-            }
+              }
-          ];
+              {
+                name = "test-drv";
+                pos = __curPos;
+                expected = {a = "b";};
+                actualDrv = pkgs.runCommand "test-something" {} ''
+                  echo "Simulating taking some time"
+                  sleep 1
+                  echo '{"a":"b"}' > $out
+                '';
+              }
+              {
+                name = "skipped";
+                pos = __curPos;
+                expected = null;
+                actual = null;
+              }
+            ];
+          };
         };
 
         ci = {

lib/flakeModule.nix

@@ -14,27 +14,39 @@ in {
     }: let
       nixtests-lib = import ./. {inherit pkgs self;};
     in {
-      options.testSuites = mkOption {
+      options.nixtest = mkOption {
-        type = types.attrsOf (types.listOf types.attrs);
+        type = types.submodule ({...}: {
+          options = {
+            skip = mkOption {
+              type = types.str;
+              default = "";
+              description = "Which tests to skip (regex)";
+            };
+            suites = mkOption {
+              type = types.attrsOf (types.listOf types.attrs);
+              default = {};
+            };
+          };
+        });
         default = {};
       };
 
       config.legacyPackages = rec {
         "nixtests" = let
           suites = map (suiteName: let
-            tests = builtins.getAttr suiteName config.testSuites;
+            tests = builtins.getAttr suiteName config.nixtest.suites;
           in
             nixtests-lib.mkSuite
             suiteName
             (map (test: nixtests-lib.mkTest test) tests))
-          (builtins.attrNames config.testSuites);
+          (builtins.attrNames config.nixtest.suites);
         in
           nixtests-lib.exportSuites suites;
         "nixtests:run" = let
           program = pkgs.callPackage ./../package.nix {};
         in
           pkgs.writeShellScriptBin "nixtests:run" ''
-            ${program}/bin/nixtest --tests=${nixtests} "$@"
+            ${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
           '';
       };
     }