From 0a1bbae2c30e3ba8e3b02de223e199c5dfd56572 Mon Sep 17 00:00:00 2001 From: technofab Date: Sun, 4 May 2025 21:54:17 +0200 Subject: [PATCH] feat: general improvements and add junit "error" and "skipped" support --- cmd/nixtest/display.go | 25 +++++---- cmd/nixtest/junit.go | 33 ++++++++++-- cmd/nixtest/main.go | 93 +++++++++++++++++++-------------- flake.nix | 113 ++++++++++++++++++++++++----------------- lib/flakeModule.nix | 22 ++++++-- 5 files changed, 181 insertions(+), 105 deletions(-) diff --git a/cmd/nixtest/display.go b/cmd/nixtest/display.go index 09fef49..5f9d441 100644 --- a/cmd/nixtest/display.go +++ b/cmd/nixtest/display.go @@ -8,20 +8,23 @@ import ( "github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/text" "github.com/rs/zerolog/log" + "github.com/sergi/go-diff/diffmatchpatch" ) func printErrors(results Results) { for _, suiteResults := range results { for _, result := range suiteResults { - if result.Success { + if result.Status == StatusSuccess || result.Status == StatusSkipped { continue } - fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Name)) + fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Spec.Name)) var message string - if result.Error.Diff != "" { - message = fmt.Sprintf("Diff:\n%s", result.Error.Diff) + if result.Status == StatusFailure { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(result.Expected, result.Actual, false) + message = fmt.Sprintf("Diff:\n%s", dmp.DiffPrettyText(diffs)) } else { - message = result.Error.Message + message = result.ErrorMessage } for line := range strings.Lines(message) { fmt.Printf("%s %s", text.FgRed.Sprint("|"), line) @@ -47,7 +50,7 @@ func printSummary(results Results, successCount int, totalCount int) { suiteSuccess := 0 for _, res := range suiteResults { - if res.Success { + if res.Status == StatusSuccess || res.Status == StatusSkipped { suiteSuccess++ } } @@ -60,15 +63,19 @@ func printSummary(results Results, successCount int, totalCount int) { }) for _, res := range suiteResults { symbol := "❌" - if res.Success { + if res.Status == StatusSuccess { symbol = "✅" + } else if res.Status == StatusError { + symbol = "error" + } else if res.Status == StatusSkipped { + symbol = "skipped" } t.AppendRow([]any{ - res.Name, + res.Spec.Name, fmt.Sprintf("%s", res.Duration), symbol, - res.Pos, + res.Spec.Pos, }) } t.AppendSeparator() diff --git a/cmd/nixtest/junit.go b/cmd/nixtest/junit.go index e9df6e6..4819482 100644 --- a/cmd/nixtest/junit.go +++ b/cmd/nixtest/junit.go @@ -13,6 +13,8 @@ type JUnitReport struct { Name string `xml:"name,attr"` Tests int `xml:"tests,attr"` Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` Time string `xml:"time,attr"` // in seconds Suites []JUnitTestSuite `xml:"testsuite"` } @@ -22,6 +24,8 @@ type JUnitTestSuite struct { Name string `xml:"name,attr"` Tests int `xml:"tests,attr"` Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` Time string `xml:"time,attr"` // in seconds TestCases []JUnitCase `xml:"testcase"` } @@ -34,6 +38,7 @@ type JUnitCase struct { Line string `xml:"line,attr,omitempty"` Failure *string `xml:"failure,omitempty"` Error *string `xml:"error,omitempty"` + Skipped *string `xml:"skipped,omitempty"` } func GenerateJUnitReport(name string, results Results) (string, error) { @@ -41,6 +46,8 @@ func GenerateJUnitReport(name string, results Results) (string, error) { Name: name, Tests: 0, Failures: 0, + Errors: 0, + Skipped: 0, Suites: []JUnitTestSuite{}, } @@ -51,6 +58,8 @@ func GenerateJUnitReport(name string, results Results) (string, error) { Name: suiteName, Tests: len(suiteResults), Failures: 0, + Errors: 0, + Skipped: 0, TestCases: []JUnitCase{}, } @@ -62,21 +71,35 @@ func GenerateJUnitReport(name string, results Results) (string, error) { suiteDuration += result.Duration testCase := JUnitCase{ - Name: result.Name, + Name: result.Spec.Name, Classname: suiteName, // Use suite name as classname Time: durationSeconds, } - if result.Pos != "" { - pos := strings.Split(result.Pos, ":") + if result.Spec.Pos != "" { + pos := strings.Split(result.Spec.Pos, ":") testCase.File = pos[0] testCase.Line = pos[1] } - if !result.Success { + if result.Status == StatusFailure { suite.Failures++ report.Failures++ - testCase.Failure = &result.Error.Message + message := fmt.Sprintf( + "Expected:\n%s\nGot:\n%s", + PrefixLines(result.Expected), + PrefixLines(result.Actual), + ) + testCase.Failure = &message + } else if result.Status == StatusError { + suite.Errors++ + report.Errors++ + testCase.Error = &result.ErrorMessage + } else if result.Status == StatusSkipped { + suite.Skipped++ + report.Skipped++ + skipped := "" + testCase.Skipped = &skipped } suite.TestCases = append(suite.TestCases, testCase) diff --git a/cmd/nixtest/main.go b/cmd/nixtest/main.go index e84496e..b48f7a3 100644 --- a/cmd/nixtest/main.go +++ b/cmd/nixtest/main.go @@ -9,13 +9,13 @@ import ( "os/exec" "path" "reflect" + "regexp" "strings" "sync" "time" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/sergi/go-diff/diffmatchpatch" flag "github.com/spf13/pflag" ) @@ -36,13 +36,22 @@ type TestSpec struct { Suite string } +type TestStatus int + +const ( + StatusSuccess TestStatus = iota + StatusFailure + StatusError + StatusSkipped +) + type TestResult struct { - Name string - Success bool - Error TestResultError - Duration time.Duration - Pos string - Suite string + Spec TestSpec + Status TestStatus + Duration time.Duration + ErrorMessage string + Expected string + Actual string } type Results map[string][]TestResult @@ -80,13 +89,6 @@ func buildAndParse(derivation string) (any, error) { return result, nil } -type TestResultError struct { - Message string - Expected string - Actual string - Diff string -} - func PrefixLines(input string) string { lines := strings.Split(input, "\n") for i := range lines { @@ -95,14 +97,29 @@ func PrefixLines(input string) string { return strings.Join(lines, "\n") } +func shouldSkip(input string, pattern string) bool { + if pattern == "" { + return false + } + + regex, err := regexp.Compile(pattern) + if err != nil { + log.Panic().Err(err).Msg("failed to compile skip regex") + } + + return regex.MatchString(input) +} + func runTest(spec TestSpec) TestResult { startTime := time.Now() result := TestResult{ - Name: spec.Name, - Pos: spec.Pos, - Suite: spec.Suite, - Success: false, - Error: TestResultError{}, + Spec: spec, + Status: StatusSuccess, + } + + if shouldSkip(spec.Name, *skipPattern) { + result.Status = StatusSkipped + return result } var actual any @@ -112,7 +129,8 @@ func runTest(spec TestSpec) TestResult { var err error actual, err = buildAndParse(spec.ActualDrv) if err != nil { - result.Error.Message = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error()) + result.Status = StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error()) goto end } } else { @@ -129,14 +147,16 @@ func runTest(spec TestSpec) TestResult { } if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) { - result.Error.Message = "No Snapshot exists yet" + result.Status = StatusError + result.ErrorMessage = "No Snapshot exists yet" goto end } var err error expected, err = ParseFile[any](filePath) if err != nil { - result.Error.Message = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error()) + result.Status = StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error()) goto end } } else if spec.Type == "unit" { @@ -146,28 +166,24 @@ func runTest(spec TestSpec) TestResult { } if reflect.DeepEqual(actual, expected) { - result.Success = true + result.Status = StatusSuccess } else { - dmp := diffmatchpatch.New() text1, err := json.MarshalIndent(expected, "", " ") if err != nil { - result.Error.Message = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error()) + result.Status = StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error()) goto end } text2, err := json.MarshalIndent(actual, "", " ") if err != nil { - result.Error.Message = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error()) + result.Status = StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error()) goto end } - diffs := dmp.DiffMain(string(text1), string(text2), false) - result.Error.Expected = string(text1) - result.Error.Actual = string(text2) - result.Error.Diff = dmp.DiffPrettyText(diffs) - result.Error.Message = fmt.Sprintf( - "Expected:\n%s\nGot:\n%s", - PrefixLines(string(text1)), - PrefixLines(string(text2)), - ) + + result.Status = StatusFailure + result.Expected = string(text1) + result.Actual = string(text2) } end: @@ -211,7 +227,8 @@ var ( junitPath *string = flag.String( "junit", "", "Path to generate JUNIT report to, leave empty to disable", ) - updateSnapshots *bool = flag.Bool("update-snapshots", false, "Update all snapshots") + updateSnapshots *bool = flag.Bool("update-snapshots", false, "Update all snapshots") + skipPattern *string = flag.String("skip", "", "Regular expression to skip (e.g., 'test-.*|.*-b')") ) func main() { @@ -269,8 +286,8 @@ func main() { successCount := 0 for r := range resultsChan { - results[r.Suite] = append(results[r.Suite], r) - if r.Success { + results[r.Spec.Suite] = append(results[r.Spec.Suite], r) + if r.Status == StatusSuccess || r.Status == StatusSkipped { successCount++ } } diff --git a/flake.nix b/flake.nix index a1c5ab2..30f637f 100644 --- a/flake.nix +++ b/flake.nix @@ -41,54 +41,71 @@ }; }; - testSuites = { - "suite-one" = [ - { - name = "test-one"; - # required to figure out file and line, but optional - pos = __curPos; - expected = 1; - actual = 1; - } - { - name = "fail"; - pos = __curPos; - expected = 0; - actual = "meow"; - } - { - name = "snapshot-test"; - type = "snapshot"; - pos = __curPos; - actual = "test"; - } - { - name = "test-snapshot-drv"; - type = "snapshot"; - pos = __curPos; - actualDrv = pkgs.runCommand "test-snapshot" {} '' - echo '"snapshot drv"' > $out - ''; - } - ]; - "other-suite" = [ - { - name = "obj-snapshot"; - type = "snapshot"; - pos = __curPos; - actual = {hello = "world";}; - } - { - name = "test-drv"; - pos = __curPos; - expected = {a = "b";}; - actualDrv = pkgs.runCommand "test-something" {} '' - echo "Simulating taking some time" - sleep 1 - echo '{"a":"b"}' > $out - ''; - } - ]; + nixtest = { + skip = "skip.*d"; + suites = { + "suite-one" = [ + { + name = "test-one"; + # required to figure out file and line, but optional + pos = __curPos; + expected = 1; + actual = 1; + } + { + name = "fail"; + pos = __curPos; + expected = 0; + actual = "meow"; + } + { + name = "snapshot-test"; + type = "snapshot"; + pos = __curPos; + actual = "test"; + } + { + name = "test-snapshot-drv"; + type = "snapshot"; + pos = __curPos; + actualDrv = pkgs.runCommand "test-snapshot" {} '' + echo '"snapshot drv"' > $out + ''; + } + { + name = "test-error-drv"; + pos = __curPos; + expected = null; + actualDrv = pkgs.runCommand "test-error-drv" {} '' + exit 1 + ''; + } + ]; + "other-suite" = [ + { + name = "obj-snapshot"; + type = "snapshot"; + pos = __curPos; + actual = {hello = "world";}; + } + { + name = "test-drv"; + pos = __curPos; + expected = {a = "b";}; + actualDrv = pkgs.runCommand "test-something" {} '' + echo "Simulating taking some time" + sleep 1 + echo '{"a":"b"}' > $out + ''; + } + { + name = "skipped"; + pos = __curPos; + expected = null; + actual = null; + } + ]; + }; }; ci = { diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 0d09608..d509751 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -14,27 +14,39 @@ in { }: let nixtests-lib = import ./. {inherit pkgs self;}; in { - options.testSuites = mkOption { - type = types.attrsOf (types.listOf types.attrs); + options.nixtest = mkOption { + type = types.submodule ({...}: { + options = { + skip = mkOption { + type = types.str; + default = ""; + description = "Which tests to skip (regex)"; + }; + suites = mkOption { + type = types.attrsOf (types.listOf types.attrs); + default = {}; + }; + }; + }); default = {}; }; config.legacyPackages = rec { "nixtests" = let suites = map (suiteName: let - tests = builtins.getAttr suiteName config.testSuites; + tests = builtins.getAttr suiteName config.nixtest.suites; in nixtests-lib.mkSuite suiteName (map (test: nixtests-lib.mkTest test) tests)) - (builtins.attrNames config.testSuites); + (builtins.attrNames config.nixtest.suites); in nixtests-lib.exportSuites suites; "nixtests:run" = let program = pkgs.callPackage ./../package.nix {}; in pkgs.writeShellScriptBin "nixtests:run" '' - ${program}/bin/nixtest --tests=${nixtests} "$@" + ${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@" ''; }; }