technofab/nixtest
1
0
Fork 0
mirror of https://gitlab.com/TECHNOFAB/nixtest.git synced 2025-12-12 02:00:18 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: general improvements and add junit "error" and "skipped" support

Browse source
This commit is contained in:
technofab 2025-05-04 21:54:17 +02:00
parent 5ae5c2dd45
commit 0a1bbae2c3
5 changed files with 181 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

25
cmd/nixtest/display.go
View file

@ -8,20 +8,23 @@ import (
"github.com/jedib0t/go-pretty/v6/table"
"github.com/jedib0t/go-pretty/v6/text"
"github.com/rs/zerolog/log"
"github.com/sergi/go-diff/diffmatchpatch"
)
func printErrors(results Results) {
for _, suiteResults := range results {
for _, result := range suiteResults {
if result.Success {
if result.Status == StatusSuccess || result.Status == StatusSkipped {
continue
}
fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Name))
fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Spec.Name))
var message string
if result.Error.Diff != "" {
message = fmt.Sprintf("Diff:\n%s", result.Error.Diff)
if result.Status == StatusFailure {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(result.Expected, result.Actual, false)
message = fmt.Sprintf("Diff:\n%s", dmp.DiffPrettyText(diffs))
} else {
message = result.Error.Message
message = result.ErrorMessage
}
for line := range strings.Lines(message) {
fmt.Printf("%s %s", text.FgRed.Sprint("|"), line)
@ -47,7 +50,7 @@ func printSummary(results Results, successCount int, totalCount int) {
suiteSuccess := 0
for _, res := range suiteResults {
if res.Success {
if res.Status == StatusSuccess || res.Status == StatusSkipped {
suiteSuccess++
}
}
@ -60,15 +63,19 @@ func printSummary(results Results, successCount int, totalCount int) {
})
for _, res := range suiteResults {
symbol := "❌"
if res.Success {
if res.Status == StatusSuccess {
symbol = "✅"
} else if res.Status == StatusError {
symbol = "error"
} else if res.Status == StatusSkipped {
symbol = "skipped"
}
t.AppendRow([]any{
res.Name,
res.Spec.Name,
fmt.Sprintf("%s", res.Duration),
symbol,
res.Pos,
res.Spec.Pos,
})
}
t.AppendSeparator()

33
cmd/nixtest/junit.go
View file

@ -13,6 +13,8 @@ type JUnitReport struct {
Name string `xml:"name,attr"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Errors int `xml:"errors,attr"`
Skipped int `xml:"skipped,attr"`
Time string `xml:"time,attr"` // in seconds
Suites []JUnitTestSuite `xml:"testsuite"`
}
@ -22,6 +24,8 @@ type JUnitTestSuite struct {
Name string `xml:"name,attr"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Errors int `xml:"errors,attr"`
Skipped int `xml:"skipped,attr"`
Time string `xml:"time,attr"` // in seconds
TestCases []JUnitCase `xml:"testcase"`
}
@ -34,6 +38,7 @@ type JUnitCase struct {
Line string `xml:"line,attr,omitempty"`
Failure *string `xml:"failure,omitempty"`
Error *string `xml:"error,omitempty"`
Skipped *string `xml:"skipped,omitempty"`
}
func GenerateJUnitReport(name string, results Results) (string, error) {
@ -41,6 +46,8 @@ func GenerateJUnitReport(name string, results Results) (string, error) {
Name: name,
Tests: 0,
Failures: 0,
Errors: 0,
Skipped: 0,
Suites: []JUnitTestSuite{},
}
@ -51,6 +58,8 @@ func GenerateJUnitReport(name string, results Results) (string, error) {
Name: suiteName,
Tests: len(suiteResults),
Failures: 0,
Errors: 0,
Skipped: 0,
TestCases: []JUnitCase{},
}
@ -62,21 +71,35 @@ func GenerateJUnitReport(name string, results Results) (string, error) {
suiteDuration += result.Duration
testCase := JUnitCase{
Name: result.Name,
Name: result.Spec.Name,
Classname: suiteName, // Use suite name as classname
Time: durationSeconds,
}
if result.Pos != "" {
pos := strings.Split(result.Pos, ":")
if result.Spec.Pos != "" {
pos := strings.Split(result.Spec.Pos, ":")
testCase.File = pos[0]
testCase.Line = pos[1]
}
if !result.Success {
if result.Status == StatusFailure {
suite.Failures++
report.Failures++
testCase.Failure = &result.Error.Message
message := fmt.Sprintf(
"Expected:\n%s\nGot:\n%s",
PrefixLines(result.Expected),
PrefixLines(result.Actual),
)
testCase.Failure = &message
} else if result.Status == StatusError {
suite.Errors++
report.Errors++
testCase.Error = &result.ErrorMessage
} else if result.Status == StatusSkipped {
suite.Skipped++
report.Skipped++
skipped := ""
testCase.Skipped = &skipped
}
suite.TestCases = append(suite.TestCases, testCase)

89
cmd/nixtest/main.go
View file

@ -9,13 +9,13 @@ import (
"os/exec"
"path"
"reflect"
"regexp"
"strings"
"sync"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/sergi/go-diff/diffmatchpatch"
flag "github.com/spf13/pflag"
)
@ -36,13 +36,22 @@ type TestSpec struct {
Suite string
}
type TestStatus int
const (
StatusSuccess TestStatus = iota
StatusFailure
StatusError
StatusSkipped
)
type TestResult struct {
Name string
Success bool
Error TestResultError
Spec TestSpec
Status TestStatus
Duration time.Duration
Pos string
Suite string
ErrorMessage string
Expected string
Actual string
}
type Results map[string][]TestResult
@ -80,13 +89,6 @@ func buildAndParse(derivation string) (any, error) {
return result, nil
}
type TestResultError struct {
Message string
Expected string
Actual string
Diff string
}
func PrefixLines(input string) string {
lines := strings.Split(input, "\n")
for i := range lines {
@ -95,14 +97,29 @@ func PrefixLines(input string) string {
return strings.Join(lines, "\n")
}
func shouldSkip(input string, pattern string) bool {
if pattern == "" {
return false
}
regex, err := regexp.Compile(pattern)
if err != nil {
log.Panic().Err(err).Msg("failed to compile skip regex")
}
return regex.MatchString(input)
}
func runTest(spec TestSpec) TestResult {
startTime := time.Now()
result := TestResult{
Name: spec.Name,
Pos: spec.Pos,
Suite: spec.Suite,
Success: false,
Error: TestResultError{},
Spec: spec,
Status: StatusSuccess,
}
if shouldSkip(spec.Name, *skipPattern) {
result.Status = StatusSkipped
return result
}
var actual any
@ -112,7 +129,8 @@ func runTest(spec TestSpec) TestResult {
var err error
actual, err = buildAndParse(spec.ActualDrv)
if err != nil {
result.Error.Message = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error())
result.Status = StatusError
result.ErrorMessage = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error())
goto end
}
} else {
@ -129,14 +147,16 @@ func runTest(spec TestSpec) TestResult {
}
if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
result.Error.Message = "No Snapshot exists yet"
result.Status = StatusError
result.ErrorMessage = "No Snapshot exists yet"
goto end
}
var err error
expected, err = ParseFile[any](filePath)
if err != nil {
result.Error.Message = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error())
result.Status = StatusError
result.ErrorMessage = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error())
goto end
}
} else if spec.Type == "unit" {
@ -146,28 +166,24 @@ func runTest(spec TestSpec) TestResult {
}
if reflect.DeepEqual(actual, expected) {
result.Success = true
result.Status = StatusSuccess
} else {
dmp := diffmatchpatch.New()
text1, err := json.MarshalIndent(expected, "", " ")
if err != nil {
result.Error.Message = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error())
result.Status = StatusError
result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error())
goto end
}
text2, err := json.MarshalIndent(actual, "", " ")
if err != nil {
result.Error.Message = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error())
result.Status = StatusError
result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error())
goto end
}
diffs := dmp.DiffMain(string(text1), string(text2), false)
result.Error.Expected = string(text1)
result.Error.Actual = string(text2)
result.Error.Diff = dmp.DiffPrettyText(diffs)
result.Error.Message = fmt.Sprintf(
"Expected:\n%s\nGot:\n%s",
PrefixLines(string(text1)),
PrefixLines(string(text2)),
)
result.Status = StatusFailure
result.Expected = string(text1)
result.Actual = string(text2)
}
end:
@ -212,6 +228,7 @@ var (
"junit", "", "Path to generate JUNIT report to, leave empty to disable",
)
updateSnapshots *bool = flag.Bool("update-snapshots", false, "Update all snapshots")
skipPattern *string = flag.String("skip", "", "Regular expression to skip (e.g., 'test-.*|.*-b')")
)
func main() {
@ -269,8 +286,8 @@ func main() {
successCount := 0
for r := range resultsChan {
results[r.Suite] = append(results[r.Suite], r)
if r.Success {
results[r.Spec.Suite] = append(results[r.Spec.Suite], r)
if r.Status == StatusSuccess || r.Status == StatusSkipped {
successCount++
}
}

19
flake.nix
View file

@ -41,7 +41,9 @@
};
};
testSuites = {
nixtest = {
skip = "skip.*d";
suites = {
"suite-one" = [
{
name = "test-one";
@ -70,6 +72,14 @@
echo '"snapshot drv"' > $out
'';
}
{
name = "test-error-drv";
pos = __curPos;
expected = null;
actualDrv = pkgs.runCommand "test-error-drv" {} ''
exit 1
'';
}
];
"other-suite" = [
{
@ -88,8 +98,15 @@
echo '{"a":"b"}' > $out
'';
}
{
name = "skipped";
pos = __curPos;
expected = null;
actual = null;
}
];
};
};
ci = {
stages = ["test"];

20
lib/flakeModule.nix
View file

@ -14,27 +14,39 @@ in {
}: let
nixtests-lib = import ./. {inherit pkgs self;};
in {
options.testSuites = mkOption {
options.nixtest = mkOption {
type = types.submodule ({...}: {
options = {
skip = mkOption {
type = types.str;
default = "";
description = "Which tests to skip (regex)";
};
suites = mkOption {
type = types.attrsOf (types.listOf types.attrs);
default = {};
};
};
});
default = {};
};
config.legacyPackages = rec {
"nixtests" = let
suites = map (suiteName: let
tests = builtins.getAttr suiteName config.testSuites;
tests = builtins.getAttr suiteName config.nixtest.suites;
in
nixtests-lib.mkSuite
suiteName
(map (test: nixtests-lib.mkTest test) tests))
(builtins.attrNames config.testSuites);
(builtins.attrNames config.nixtest.suites);
in
nixtests-lib.exportSuites suites;
"nixtests:run" = let
program = pkgs.callPackage ./../package.nix {};
in
pkgs.writeShellScriptBin "nixtests:run" ''
${program}/bin/nixtest --tests=${nixtests} "$@"
${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
'';
};
}