diff --git a/.gitignore b/.gitignore index 247d87e..e1f66c8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ result .pre-commit-config.yaml *.xml +cover.* diff --git a/cmd/nixtest/display.go b/cmd/nixtest/display.go deleted file mode 100644 index de6c8b6..0000000 --- a/cmd/nixtest/display.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "fmt" - "maps" - "os" - "regexp" - "slices" - "sort" - "strings" - - "github.com/jedib0t/go-pretty/v6/table" - "github.com/jedib0t/go-pretty/v6/text" - "github.com/rs/zerolog/log" - "github.com/sergi/go-diff/diffmatchpatch" -) - -func printErrors(results Results) { - for _, suiteResults := range results { - for _, result := range suiteResults { - if result.Status == StatusSuccess || result.Status == StatusSkipped { - continue - } - fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Spec.Name)) - var message string = result.ErrorMessage - // if ErrorMessage is set, prefer that - if result.Status == StatusFailure && message == "" { - dmp := diffmatchpatch.New() - diffs := dmp.DiffMain(result.Expected, result.Actual, false) - message = fmt.Sprintf("Diff:\n%s", dmp.DiffPrettyText(diffs)) - } - - endedWithNewline := false - // handle multi-line colored changes - colorState := "" - colorRegex := regexp.MustCompile(`\x1b\[[0-9;]*m`) // Match any escape sequence - for line := range strings.Lines(message) { - coloredLine := colorState + line - endedWithNewline = strings.HasSuffix(coloredLine, "\n") - fmt.Printf("%s %s", text.FgRed.Sprint("|"), coloredLine) - - matches := colorRegex.FindAllString(line, -1) - - // determine last color code, to copy to next line - if len(matches) > 0 { - lastMatch := matches[len(matches)-1] - if lastMatch == "\x1b[0m" { - colorState = "" // reset color state - } else { - colorState = lastMatch // save color state for next line - } - } - } - if endedWithNewline { - fmt.Printf("%s", text.FgRed.Sprint("|")) - } - if message == "" { - fmt.Printf("- no output -") - } - fmt.Printf("\n\n") - } - } -} - -func printSummary(results Results, successCount int, totalCount int) { - t := table.NewWriter() - t.SetStyle(table.StyleLight) - t.SetOutputMirror(os.Stdout) - t.AppendHeader(table.Row{"Test", "Duration", "Pass", "File"}) - - log.Info().Msg("Summary:") - - suitesSorted := slices.Sorted(maps.Keys(results)) - for _, suite := range suitesSorted { - suiteResults := results[suite] - suiteTotal := len(suiteResults) - suiteSuccess := 0 - - for _, res := range suiteResults { - if res.Status == StatusSuccess || res.Status == StatusSkipped { - suiteSuccess++ - } - } - - t.AppendRow(table.Row{ - text.Bold.Sprint(suite), - "", - fmt.Sprintf("%d/%d", suiteSuccess, suiteTotal), - "", - }) - sort.Slice(suiteResults, func(i, j int) bool { - return suiteResults[i].Spec.Name < suiteResults[j].Spec.Name - }) - for _, res := range suiteResults { - symbol := "❌" - if res.Status == StatusSuccess { - symbol = "✅" - } else if res.Status == StatusError { - symbol = "error" - } else if res.Status == StatusSkipped { - symbol = "skipped" - } - - t.AppendRow([]any{ - res.Spec.Name, - fmt.Sprintf("%s", res.Duration), - symbol, - res.Spec.Pos, - }) - } - t.AppendSeparator() - } - - t.AppendFooter(table.Row{ - text.Bold.Sprint("TOTAL"), - "", - fmt.Sprintf("%d/%d", successCount, totalCount), - "", - }) - t.Render() -} diff --git a/cmd/nixtest/junit.go b/cmd/nixtest/junit.go deleted file mode 100644 index 4a4a1ca..0000000 --- a/cmd/nixtest/junit.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "encoding/xml" - "fmt" - "os" - "strings" - "time" - - "github.com/akedrou/textdiff" - "github.com/akedrou/textdiff/myers" -) - -type JUnitReport struct { - XMLName xml.Name `xml:"testsuite"` - Name string `xml:"name,attr"` - Tests int `xml:"tests,attr"` - Failures int `xml:"failures,attr"` - Errors int `xml:"errors,attr"` - Skipped int `xml:"skipped,attr"` - Time string `xml:"time,attr"` // in seconds - Suites []JUnitTestSuite `xml:"testsuite"` -} - -type JUnitTestSuite struct { - XMLName xml.Name `xml:"testsuite"` - Name string `xml:"name,attr"` - Tests int `xml:"tests,attr"` - Failures int `xml:"failures,attr"` - Errors int `xml:"errors,attr"` - Skipped int `xml:"skipped,attr"` - Time string `xml:"time,attr"` // in seconds - TestCases []JUnitCase `xml:"testcase"` -} - -type JUnitCase struct { - Name string `xml:"name,attr"` - Classname string `xml:"classname,attr"` - Time string `xml:"time,attr"` // in seconds - File string `xml:"file,attr,omitempty"` - Line string `xml:"line,attr,omitempty"` - Failure *string `xml:"failure,omitempty"` - Error *string `xml:"error,omitempty"` - Skipped *string `xml:"skipped,omitempty"` -} - -func GenerateJUnitReport(name string, results Results) (string, error) { - report := JUnitReport{ - Name: name, - Tests: 0, - Failures: 0, - Errors: 0, - Skipped: 0, - Suites: []JUnitTestSuite{}, - } - - totalDuration := time.Duration(0) - - for suiteName, suiteResults := range results { - suite := JUnitTestSuite{ - Name: suiteName, - Tests: len(suiteResults), - Failures: 0, - Errors: 0, - Skipped: 0, - TestCases: []JUnitCase{}, - } - - suiteDuration := time.Duration(0) - - for _, result := range suiteResults { - durationSeconds := fmt.Sprintf("%.3f", result.Duration.Seconds()) - totalDuration += result.Duration - suiteDuration += result.Duration - - testCase := JUnitCase{ - Name: result.Spec.Name, - Classname: suiteName, // Use suite name as classname - Time: durationSeconds, - } - - if result.Spec.Pos != "" { - pos := strings.Split(result.Spec.Pos, ":") - testCase.File = pos[0] - testCase.Line = pos[1] - } - - if result.Status == StatusFailure { - suite.Failures++ - report.Failures++ - if result.ErrorMessage != "" { - testCase.Failure = &result.ErrorMessage - } else { - // FIXME: ComputeEdits deprecated - edits := myers.ComputeEdits(result.Expected, result.Actual) - diff, err := textdiff.ToUnified("expected", "actual", result.Expected, edits, 3) - if err != nil { - return "", err - } - // remove newline hint - diff = strings.ReplaceAll(diff, "\\ No newline at end of file\n", "") - testCase.Failure = &diff - } - } else if result.Status == StatusError { - suite.Errors++ - report.Errors++ - testCase.Error = &result.ErrorMessage - } else if result.Status == StatusSkipped { - suite.Skipped++ - report.Skipped++ - skipped := "" - testCase.Skipped = &skipped - } - - suite.TestCases = append(suite.TestCases, testCase) - } - suite.Time = fmt.Sprintf("%.3f", suiteDuration.Seconds()) - report.Suites = append(report.Suites, suite) - report.Tests += len(suiteResults) - } - - report.Time = fmt.Sprintf("%.3f", totalDuration.Seconds()) - - output, err := xml.MarshalIndent(report, "", " ") - if err != nil { - return "", fmt.Errorf("failed to marshal XML: %w", err) - } - - return xml.Header + string(output), nil -} - -func GenerateJunitFile(path string, results Results) error { - res, err := GenerateJUnitReport("nixtest", results) - if err != nil { - return fmt.Errorf("failed to generate junit report: %w", err) - } - file, err := os.Create(*junitPath) - if err != nil { - return fmt.Errorf("failed to create file: %w", err) - } - defer file.Close() - - _, err = file.WriteString(res) - if err != nil { - return fmt.Errorf("failed to write to file: %w", err) - } - return nil -} diff --git a/cmd/nixtest/main.go b/cmd/nixtest/main.go index e092788..5f141c1 100644 --- a/cmd/nixtest/main.go +++ b/cmd/nixtest/main.go @@ -1,385 +1,100 @@ package main import ( - "bytes" - "encoding/json" - "errors" - "fmt" "os" - "os/exec" - "path" - "reflect" - "regexp" - "strings" - "sync" - "time" + + "gitlab.com/technofab/nixtest/internal/config" + appnix "gitlab.com/technofab/nixtest/internal/nix" + "gitlab.com/technofab/nixtest/internal/report/console" + "gitlab.com/technofab/nixtest/internal/report/junit" + "gitlab.com/technofab/nixtest/internal/runner" + appsnap "gitlab.com/technofab/nixtest/internal/snapshot" + "gitlab.com/technofab/nixtest/internal/types" + "gitlab.com/technofab/nixtest/internal/util" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - flag "github.com/spf13/pflag" -) - -type SuiteSpec struct { - Name string `json:"name"` - Tests []TestSpec `json:"tests"` -} - -type TestSpec struct { - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - Expected any `json:"expected,omitempty"` - Actual any `json:"actual,omitempty"` - ActualDrv string `json:"actualDrv,omitempty"` - Script string `json:"script,omitempty"` - Pos string `json:"pos,omitempty"` - - Suite string -} - -type TestStatus int - -const ( - StatusSuccess TestStatus = iota - StatusFailure - StatusError - StatusSkipped -) - -type TestResult struct { - Spec TestSpec - Status TestStatus - Duration time.Duration - ErrorMessage string - Expected string - Actual string -} - -type Results map[string][]TestResult - -func buildDerivation(derivation string) (string, error) { - cmd := exec.Command( - "nix", - "build", - derivation+"^*", - "--print-out-paths", - "--no-link", - ) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - if err != nil { - return "", fmt.Errorf("failed to run nix build: %v, %s", err, stderr.String()) - } - - path := strings.TrimSpace(stdout.String()) - return path, nil -} - -func buildAndParse(derivation string) (any, error) { - path, err := buildDerivation(derivation) - if err != nil { - return nil, err - } - - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var result any - err = json.Unmarshal(data, &result) - if err != nil { - return nil, err - } - - return result, nil -} - -// builds derivation and runs it -func buildAndRun(derivation string) (exitCode int, stdout bytes.Buffer, stderr bytes.Buffer, err error) { - exitCode = -1 - path, err := buildDerivation(derivation) - if err != nil { - return - } - - args := []string{"bash", path} - if *pure { - args = append([]string{"env", "-i"}, args...) - } - cmd := exec.Command(args[0], args[1:]...) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err = cmd.Start(); err != nil { - return - } - - if err = cmd.Wait(); err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - exitCode = exiterr.ExitCode() - err = nil - } - return - } - - return 0, stdout, stderr, nil -} - -func PrefixLines(input string) string { - lines := strings.Split(input, "\n") - for i := range lines { - lines[i] = "| " + lines[i] - } - return strings.Join(lines, "\n") -} - -func shouldSkip(input string, pattern string) bool { - if pattern == "" { - return false - } - - regex, err := regexp.Compile(pattern) - if err != nil { - log.Panic().Err(err).Msg("failed to compile skip regex") - } - - return regex.MatchString(input) -} - -func isString(value any) bool { - switch value.(type) { - case string: - return true - default: - return false - } -} - -func runTest(spec TestSpec) TestResult { - startTime := time.Now() - result := TestResult{ - Spec: spec, - Status: StatusSuccess, - } - - if shouldSkip(spec.Name, *skipPattern) { - result.Status = StatusSkipped - return result - } - - var actual any - var expected any - - if spec.ActualDrv != "" { - var err error - actual, err = buildAndParse(spec.ActualDrv) - if err != nil { - result.Status = StatusError - result.ErrorMessage = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error()) - goto end - } - } else { - actual = spec.Actual - } - if spec.Type == "snapshot" { - filePath := path.Join( - *snapshotDir, - fmt.Sprintf("%s.snap.json", strings.ToLower(spec.Name)), - ) - - if *updateSnapshots { - createSnapshot(filePath, actual) - } - - if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) { - result.Status = StatusError - result.ErrorMessage = "No Snapshot exists yet" - goto end - } - - var err error - expected, err = ParseFile[any](filePath) - if err != nil { - result.Status = StatusError - result.ErrorMessage = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error()) - goto end - } - } else if spec.Type == "unit" { - expected = spec.Expected - } else if spec.Type == "script" { - exitCode, stdout, stderr, err := buildAndRun(spec.Script) - if err != nil { - result.Status = StatusError - result.ErrorMessage = fmt.Sprintf("[system] failed to run script: %v", err.Error()) - } - if exitCode != 0 { - result.Status = StatusFailure - result.ErrorMessage = fmt.Sprintf("[exit code %d]\n[stdout]\n%s\n[stderr]\n%s", exitCode, stdout.String(), stderr.String()) - } - // no need for equality checking with "script" - goto end - } else { - log.Panic().Str("type", spec.Type).Msg("Invalid test type") - } - - if reflect.DeepEqual(actual, expected) { - result.Status = StatusSuccess - } else { - var text1, text2 string - - // just keep strings as is, only json marshal if any of them is not a string - if isString(actual) && isString(expected) { - text1 = actual.(string) - text2 = expected.(string) - } else { - bytes1, err := json.MarshalIndent(expected, "", " ") - if err != nil { - result.Status = StatusError - result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error()) - goto end - } - bytes2, err := json.MarshalIndent(actual, "", " ") - if err != nil { - result.Status = StatusError - result.ErrorMessage = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error()) - goto end - } - text1 = string(bytes1) - text2 = string(bytes2) - } - - result.Status = StatusFailure - result.Expected = text1 - result.Actual = text2 - } - -end: - result.Duration = time.Since(startTime) - return result -} - -func createSnapshot(filePath string, actual any) error { - jsonData, err := json.Marshal(actual) - if err != nil { - return err - } - - err = os.MkdirAll(path.Dir(filePath), 0777) - if err != nil { - return err - } - - err = os.WriteFile(filePath, jsonData, 0644) - if err != nil { - return err - } - - return nil -} - -// worker to process TestSpec items -func worker(jobs <-chan TestSpec, results chan<- TestResult, wg *sync.WaitGroup) { - defer wg.Done() - for spec := range jobs { - results <- runTest(spec) - } -} - -var ( - numWorkers *int = flag.Int("workers", 4, "Amount of tests to run in parallel") - testsFile *string = flag.String("tests", "", "Path to JSON file containing tests") - snapshotDir *string = flag.String( - "snapshot-dir", "./snapshots", "Directory where snapshots are stored", - ) - junitPath *string = flag.String( - "junit", "", "Path to generate JUNIT report to, leave empty to disable", - ) - updateSnapshots *bool = flag.Bool("update-snapshots", false, "Update all snapshots") - skipPattern *string = flag.String("skip", "", "Regular expression to skip (e.g., 'test-.*|.*-b')") - pure *bool = flag.Bool("pure", false, "Unset all env vars before running script tests") ) func main() { - flag.Parse() + defer func() { + if r := recover(); r != nil { + log.Fatal().Any("r", r).Msg("Panicked") + } + }() + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: zerolog.TimeFieldFormat}) + zerolog.SetGlobalLevel(zerolog.InfoLevel) - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + appCfg := config.Load() log.Info(). - Int("workers", *numWorkers). + Int("workers", appCfg.NumWorkers). + Str("testsFile", appCfg.TestsFile). Msg("Starting nixtest") - if _, err := os.Stat(*testsFile); errors.Is(err, os.ErrNotExist) { - log.Error().Str("file", *testsFile).Msg("Tests file does not exist") + if _, err := os.Stat(appCfg.TestsFile); os.IsNotExist(err) { + log.Error().Str("file", appCfg.TestsFile).Msg("Tests file does not exist") os.Exit(1) } - parsedSpecs, err := ParseFile[[]SuiteSpec](*testsFile) + suites, err := util.ParseFile[[]types.SuiteSpec](appCfg.TestsFile) if err != nil { log.Error().Err(err).Msg("Failed to load tests from file") os.Exit(1) } totalTests := 0 - for _, suite := range parsedSpecs { + for _, suite := range suites { totalTests += len(suite.Tests) } log.Info(). - Int("suites", len(parsedSpecs)). + Int("suites", len(suites)). Int("tests", totalTests). - Msg("Discovered suites") + Msg("Discovered suites and tests") - jobsChan := make(chan TestSpec, totalTests) - resultsChan := make(chan TestResult, totalTests) + nixService := appnix.NewDefaultService() + snapshotService := appsnap.NewDefaultService() - var wg sync.WaitGroup - - for i := 1; i <= *numWorkers; i++ { - wg.Add(1) - go worker(jobsChan, resultsChan, &wg) + runnerCfg := runner.Config{ + NumWorkers: appCfg.NumWorkers, + SnapshotDir: appCfg.SnapshotDir, + UpdateSnapshots: appCfg.UpdateSnapshots, + SkipPattern: appCfg.SkipPattern, + PureEnv: appCfg.PureEnv, + } + testRunner, err := runner.New(runnerCfg, nixService, snapshotService) + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize test runner") } - for _, suite := range parsedSpecs { - for _, test := range suite.Tests { - test.Suite = suite.Name - jobsChan <- test - } - } - close(jobsChan) + results := testRunner.RunTests(suites) - wg.Wait() - close(resultsChan) - - results := map[string][]TestResult{} - - successCount := 0 - - for r := range resultsChan { - results[r.Spec.Suite] = append(results[r.Spec.Suite], r) - if r.Status == StatusSuccess || r.Status == StatusSkipped { - successCount++ + relevantSuccessCount := 0 + for _, suiteResults := range results { + for _, r := range suiteResults { + if r.Status == types.StatusSuccess || r.Status == types.StatusSkipped { + relevantSuccessCount++ + } } } - if *junitPath != "" { - err = GenerateJunitFile(*junitPath, results) + if appCfg.JunitPath != "" { + err = junit.WriteFile(appCfg.JunitPath, "nixtest", results) if err != nil { log.Error().Err(err).Msg("Failed to generate junit file") } else { - log.Info().Str("path", *junitPath).Msg("Generated Junit report") + log.Info().Str("path", appCfg.JunitPath).Msg("Generated Junit report") } } - // print errors/logs of failed tests - printErrors(results) + // print errors first then summary + console.PrintErrors(results, appCfg.NoColor) + console.PrintSummary(results, relevantSuccessCount, totalTests) - // show table summary - printSummary(results, successCount, totalTests) - - if successCount != totalTests { - os.Exit(2) + if relevantSuccessCount != totalTests { + log.Error().Msgf("Test run finished with failures or errors. %d/%d successful (includes skipped).", relevantSuccessCount, totalTests) + os.Exit(2) // exit 2 on test failures, 1 is for internal errors } + + log.Info().Msg("All tests passed successfully!") } diff --git a/cmd/nixtest/utils.go b/cmd/nixtest/utils.go deleted file mode 100644 index 515a938..0000000 --- a/cmd/nixtest/utils.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" -) - -func ParseFile[T any](filePath string) (result T, err error) { - file, err := os.Open(filePath) - if err != nil { - return result, fmt.Errorf("failed to open file %s: %w", filePath, err) - } - defer file.Close() - - decoder := json.NewDecoder(file) - - err = decoder.Decode(&result) - if err != nil { - return result, fmt.Errorf("failed to decode JSON from file %s: %w", filePath, err) - } - - return result, nil -} diff --git a/flake.lock b/flake.lock index e4cecf7..09366e0 100644 --- a/flake.lock +++ b/flake.lock @@ -217,6 +217,23 @@ "type": "github" } }, + "nix-devtools": { + "locked": { + "dir": "lib", + "lastModified": 1739971859, + "narHash": "sha256-DaY11jX7Lraw7mRUIsgPsO+aSkkewQe2D+WMZORTNPE=", + "owner": "technofab", + "repo": "nix-devtools", + "rev": "b4f059657de5ac2569afd69a8f042614d309e6bb", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "technofab", + "repo": "nix-devtools", + "type": "gitlab" + } + }, "nix-gitlab-ci": { "locked": { "dir": "lib", @@ -352,6 +369,7 @@ "devenv": "devenv", "flake-parts": "flake-parts_2", "mkdocs-material-umami": "mkdocs-material-umami", + "nix-devtools": "nix-devtools", "nix-gitlab-ci": "nix-gitlab-ci", "nix-mkdocs": "nix-mkdocs", "nixpkgs": "nixpkgs_4", diff --git a/flake.nix b/flake.nix index 8f90425..bcc7b54 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,7 @@ inputs.devenv.flakeModule inputs.treefmt-nix.flakeModule inputs.nix-gitlab-ci.flakeModule + inputs.nix-devtools.flakeModule inputs.nix-mkdocs.flakeModule ./lib/flakeModule.nix ]; @@ -36,7 +37,7 @@ }; devenv.shells.default = { containers = pkgs.lib.mkForce {}; - packages = [pkgs.gopls pkgs.gore]; + packages = with pkgs; [gopls gore go-junit-report]; languages.go.enable = true; @@ -47,6 +48,19 @@ }; convco.enable = true; }; + + task = { + enable = true; + alias = ","; + tasks = { + "test" = { + cmds = [ + "go test -v -coverprofile cover.out ./..." + "go tool cover -html cover.out -o cover.html" + ]; + }; + }; + }; }; nixtest = { @@ -233,6 +247,32 @@ reports.junit = "junit.xml"; }; }; + "test:go" = { + stage = "test"; + nix.deps = with pkgs; [go go-junit-report gocover-cobertura]; + variables = { + GOPATH = "$CI_PROJECT_DIR/.go"; + GOCACHE = "$CI_PROJECT_DIR/.go/pkg/mod"; + }; + script = [ + "go test -coverprofile=coverage.out -v 2>&1 ./... | go-junit-report -set-exit-code > report.xml" + "go tool cover -func coverage.out" + "gocover-cobertura < coverage.out > coverage.xml" + ]; + allow_failure = true; + coverage = "/\(statements\)(?:\s+)?(\d+(?:\.\d+)?%)/"; + cache.paths = [".go/pkg/mod/"]; + artifacts = { + when = "always"; + reports = { + junit = "report.xml"; + coverage_report = { + coverage_format = "cobertura"; + path = "coverage.xml"; + }; + }; + }; + }; "docs" = { stage = "build"; script = [ @@ -273,6 +313,7 @@ devenv.url = "github:cachix/devenv"; treefmt-nix.url = "github:numtide/treefmt-nix"; nix-gitlab-ci.url = "gitlab:technofab/nix-gitlab-ci/2.0.1?dir=lib"; + nix-devtools.url = "gitlab:technofab/nix-devtools?dir=lib"; nix-mkdocs.url = "gitlab:technofab/nixmkdocs?dir=lib"; mkdocs-material-umami.url = "gitlab:technofab/mkdocs-material-umami"; }; diff --git a/go.mod b/go.mod index a6c7a55..4cb8b73 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gitlab.com/technofab/testnix +module gitlab.com/technofab/nixtest go 1.24.2 @@ -6,12 +6,16 @@ require ( github.com/akedrou/textdiff v0.1.0 github.com/rs/zerolog v1.34.0 github.com/spf13/pflag v1.0.6 + github.com/stretchr/testify v1.10.0 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect golang.org/x/text v0.25.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 20757eb..84d30d0 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/jedib0t/go-pretty/v6 v6.6.7 h1:m+LbHpm0aIAPLzLbMfn8dc3Ht8MW7lsSO4MPItz/Uuo= github.com/jedib0t/go-pretty/v6 v6.6.7/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -44,6 +46,7 @@ golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..f0b41e6 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,43 @@ +package config + +import ( + "github.com/jedib0t/go-pretty/v6/text" + "github.com/rs/zerolog/log" + flag "github.com/spf13/pflag" +) + +type AppConfig struct { + NumWorkers int + TestsFile string + SnapshotDir string + JunitPath string + UpdateSnapshots bool + SkipPattern string + PureEnv bool + NoColor bool +} + +// loads configuration from cli flags +func Load() AppConfig { + cfg := AppConfig{} + flag.IntVarP(&cfg.NumWorkers, "workers", "w", 4, "Amount of tests to run in parallel") + flag.StringVarP(&cfg.TestsFile, "tests", "f", "", "Path to JSON file containing tests (required)") + flag.StringVar(&cfg.SnapshotDir, "snapshot-dir", "./snapshots", "Directory where snapshots are stored") + flag.StringVar(&cfg.JunitPath, "junit", "", "Path to generate JUNIT report to, leave empty to disable") + flag.BoolVarP(&cfg.UpdateSnapshots, "update-snapshots", "u", false, "Update all snapshots") + flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')") + flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests") + flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring") + + flag.Parse() + + if cfg.TestsFile == "" { + log.Panic().Msg("Tests file path (-f or --tests) is required.") + } + + if cfg.NoColor { + text.DisableColors() + } + + return cfg +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..c864d90 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,89 @@ +package config + +import ( + "os" + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" +) + +func TestLoad_Defaults(t *testing.T) { + originalArgs := os.Args + oldFlagSet := pflag.CommandLine + defer func() { + os.Args = originalArgs + pflag.CommandLine = oldFlagSet + }() + + // for Load() to not call log.Fatal(), a tests file must be provided + os.Args = []string{"cmd", "-f", "dummy.json"} + pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) // reset flags + + cfg := Load() + + if cfg.NumWorkers != 4 { + t.Errorf("Default NumWorkers: got %d, want 4", cfg.NumWorkers) + } + if cfg.SnapshotDir != "./snapshots" { + t.Errorf("Default SnapshotDir: got %s, want ./snapshots", cfg.SnapshotDir) + } +} + +func TestLoad_Fatal(t *testing.T) { + originalArgs := os.Args + oldFlagSet := pflag.CommandLine + defer func() { + os.Args = originalArgs + pflag.CommandLine = oldFlagSet + }() + + // Load() should panic without tests file + os.Args = []string{ + "cmd", + } + pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) // Reset flags + + assert.Panics(t, func() { _ = Load() }, "Load should panic withot tests file") +} + +func TestLoad_CustomValues(t *testing.T) { + originalArgs := os.Args + oldFlagSet := pflag.CommandLine + defer func() { + os.Args = originalArgs + pflag.CommandLine = oldFlagSet + }() + + // simulate cli args for Load() + os.Args = []string{ + "cmd", // dummy command name + "-f", "mytests.json", + "--workers", "8", + "--snapshot-dir", "/tmp/snaps", + "--junit", "report.xml", + "-u", + "--skip", "specific-test", + "--pure", + "--no-color", + } + pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) // Reset flags + + cfg := Load() + + if cfg.TestsFile != "mytests.json" { + t.Errorf("TestsFile: got %s, want mytests.json", cfg.TestsFile) + } + if cfg.NumWorkers != 8 { + t.Errorf("NumWorkers: got %d, want 8", cfg.NumWorkers) + } + if !cfg.UpdateSnapshots { + t.Errorf("UpdateSnapshots: got %v, want true", cfg.UpdateSnapshots) + } + if cfg.SkipPattern != "specific-test" { + t.Errorf("SkipPattern: got %s, want specific-test", cfg.SkipPattern) + } + if !cfg.PureEnv { + t.Errorf("PureEnv: got %v, want true", cfg.PureEnv) + } +} diff --git a/internal/errors/errors.go b/internal/errors/errors.go new file mode 100644 index 0000000..cb08ae4 --- /dev/null +++ b/internal/errors/errors.go @@ -0,0 +1,82 @@ +package errors + +import ( + "fmt" +) + +// NixBuildError indicates an error during `nix build` +type NixBuildError struct { + Derivation string + Stderr string + Err error // underlying error from exec.Cmd or similar +} + +func (e *NixBuildError) Error() string { + return fmt.Sprintf("nix build for %s failed: %v (stderr: %s)", e.Derivation, e.Err, e.Stderr) +} +func (e *NixBuildError) Unwrap() error { return e.Err } + +// NixNoOutputPathError indicates `nix build` succeeded but produced no output path +type NixNoOutputPathError struct { + Derivation string + Stderr string +} + +func (e *NixNoOutputPathError) Error() string { + return fmt.Sprintf("nix build for %s produced no output path (stderr: %s)", e.Derivation, e.Stderr) +} + +// FileReadError indicates an error reading a file, often a derivation output +type FileReadError struct { + Path string + Err error +} + +func (e *FileReadError) Error() string { + return fmt.Sprintf("failed to read file %s: %v", e.Path, e.Err) +} +func (e *FileReadError) Unwrap() error { return e.Err } + +// JSONUnmarshalError indicates an error unmarshalling JSON data +type JSONUnmarshalError struct { + Source string // e.g. file path or "derivation output" + Err error +} + +func (e *JSONUnmarshalError) Error() string { + return fmt.Sprintf("failed to unmarshal JSON from %s: %v", e.Source, e.Err) +} +func (e *JSONUnmarshalError) Unwrap() error { return e.Err } + +// ScriptExecutionError indicates an error starting or waiting for a script +type ScriptExecutionError struct { + Path string // path to script that was attempted to run + Err error +} + +func (e *ScriptExecutionError) Error() string { + return fmt.Sprintf("script %s execution failed: %v", e.Path, e.Err) +} +func (e *ScriptExecutionError) Unwrap() error { return e.Err } + +// SnapshotCreateError indicates an error during snapshot creation +type SnapshotCreateError struct { + FilePath string + Err error +} + +func (e *SnapshotCreateError) Error() string { + return fmt.Sprintf("failed to create/update snapshot %s: %v", e.FilePath, e.Err) +} +func (e *SnapshotCreateError) Unwrap() error { return e.Err } + +// SnapshotLoadError indicates an error loading a snapshot file +type SnapshotLoadError struct { + FilePath string + Err error +} + +func (e *SnapshotLoadError) Error() string { + return fmt.Sprintf("failed to load/parse snapshot %s: %v", e.FilePath, e.Err) +} +func (e *SnapshotLoadError) Unwrap() error { return e.Err } diff --git a/internal/errors/errors_test.go b/internal/errors/errors_test.go new file mode 100644 index 0000000..7a2872a --- /dev/null +++ b/internal/errors/errors_test.go @@ -0,0 +1,112 @@ +package errors + +import ( + "errors" + "fmt" + "os" + "testing" +) + +func TestNixBuildError(t *testing.T) { + underlyingErr := errors.New("exec: \"nix\": executable file not found in $PATH") + err := &NixBuildError{ + Derivation: "test.drv", + Stderr: "some stderr output", + Err: underlyingErr, + } + + expectedMsg := "nix build for test.drv failed: exec: \"nix\": executable file not found in $PATH (stderr: some stderr output)" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } + + if !errors.Is(err, underlyingErr) { + t.Errorf("Unwrap() failed, underlying error not found") + } +} + +func TestNixNoOutputPathError(t *testing.T) { + err := &NixNoOutputPathError{ + Derivation: "empty.drv", + Stderr: "build successful, but no paths", + } + expectedMsg := "nix build for empty.drv produced no output path (stderr: build successful, but no paths)" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } +} + +func TestFileReadError(t *testing.T) { + underlyingErr := os.ErrPermission + err := &FileReadError{ + Path: "/tmp/file.json", + Err: underlyingErr, + } + expectedMsg := "failed to read file /tmp/file.json: permission denied" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } + if !errors.Is(err, underlyingErr) { + t.Errorf("Unwrap() failed, underlying error not found") + } +} + +func TestJSONUnmarshalError(t *testing.T) { + underlyingErr := errors.New("unexpected end of JSON input") + err := &JSONUnmarshalError{ + Source: "/tmp/data.json", + Err: underlyingErr, + } + expectedMsg := "failed to unmarshal JSON from /tmp/data.json: unexpected end of JSON input" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } + if !errors.Is(err, underlyingErr) { + t.Errorf("Unwrap() failed, underlying error not found") + } +} + +func TestScriptExecutionError(t *testing.T) { + underlyingErr := errors.New("command timed out") + err := &ScriptExecutionError{ + Path: "/tmp/script.sh", + Err: underlyingErr, + } + expectedMsg := "script /tmp/script.sh execution failed: command timed out" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } + if !errors.Is(err, underlyingErr) { + t.Errorf("Unwrap() failed, underlying error not found") + } +} + +func TestSnapshotCreateError(t *testing.T) { + underlyingErr := os.ErrExist + err := &SnapshotCreateError{ + FilePath: "/snapshots/test.snap.json", + Err: underlyingErr, + } + expectedMsg := "failed to create/update snapshot /snapshots/test.snap.json: file already exists" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } + if !errors.Is(err, underlyingErr) { + t.Errorf("Unwrap() failed, underlying error not found") + } +} + +func TestSnapshotLoadError(t *testing.T) { + underlyingErr := &JSONUnmarshalError{Source: "test.snap.json", Err: fmt.Errorf("bad json")} + err := &SnapshotLoadError{ + FilePath: "/snapshots/test.snap.json", + Err: underlyingErr, + } + expectedMsg := "failed to load/parse snapshot /snapshots/test.snap.json: failed to unmarshal JSON from test.snap.json: bad json" + if err.Error() != expectedMsg { + t.Errorf("Error() got %q, want %q", err.Error(), expectedMsg) + } + if !errors.Is(err, underlyingErr) { + t.Errorf("Unwrap() failed, underlying error not found") + } +} diff --git a/internal/nix/service.go b/internal/nix/service.go new file mode 100644 index 0000000..c0ec851 --- /dev/null +++ b/internal/nix/service.go @@ -0,0 +1,110 @@ +package nix + +import ( + "bytes" + "encoding/json" + "os" + "os/exec" + "strings" + + apperrors "gitlab.com/technofab/nixtest/internal/errors" +) + +// Service defines operations related to Nix +type Service interface { + BuildDerivation(derivation string) (string, error) + BuildAndParseJSON(derivation string) (any, error) + BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) +} + +type DefaultService struct { + commandExecutor func(command string, args ...string) *exec.Cmd +} + +func NewDefaultService() *DefaultService { + return &DefaultService{commandExecutor: exec.Command} +} + +// BuildDerivation builds a Nix derivation and returns the output path +func (s *DefaultService) BuildDerivation(derivation string) (string, error) { + cmd := s.commandExecutor( + "nix", + "build", + derivation+"^*", + "--print-out-paths", + "--no-link", + ) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return "", &apperrors.NixBuildError{Derivation: derivation, Stderr: stderr.String(), Err: err} + } + + path := strings.TrimSpace(stdout.String()) + if path == "" { + return "", &apperrors.NixNoOutputPathError{Derivation: derivation, Stderr: stderr.String()} + } + return path, nil +} + +// BuildAndParseJSON builds a derivation and parses its output file as JSON +func (s *DefaultService) BuildAndParseJSON(derivation string) (any, error) { + path, err := s.BuildDerivation(derivation) + if err != nil { + return nil, err + } + + data, err := os.ReadFile(path) + if err != nil { + return nil, &apperrors.FileReadError{Path: path, Err: err} + } + + var result any + err = json.Unmarshal(data, &result) + if err != nil { + return nil, &apperrors.JSONUnmarshalError{Source: path, Err: err} + } + + return result, nil +} + +// BuildAndRunScript builds a derivation and runs it as a script +func (s *DefaultService) BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) { + exitCode = -1 + path, err := s.BuildDerivation(derivation) + if err != nil { + return exitCode, "", "", err + } + + var cmdArgs []string + if pureEnv { + cmdArgs = append([]string{"env", "-i"}, "bash", path) + } else { + cmdArgs = []string{"bash", path} + } + + cmd := s.commandExecutor(cmdArgs[0], cmdArgs[1:]...) + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + + if err = cmd.Start(); err != nil { + return exitCode, "", "", &apperrors.ScriptExecutionError{Path: path, Err: err} + } + + runErr := cmd.Wait() + stdout = outBuf.String() + stderr = errBuf.String() + + if runErr != nil { + if exitErr, ok := runErr.(*exec.ExitError); ok { + return exitErr.ExitCode(), stdout, stderr, nil + } + return exitCode, stdout, stderr, &apperrors.ScriptExecutionError{Path: path, Err: runErr} + } + + return 0, stdout, stderr, nil +} diff --git a/internal/nix/service_test.go b/internal/nix/service_test.go new file mode 100644 index 0000000..3fee50a --- /dev/null +++ b/internal/nix/service_test.go @@ -0,0 +1,305 @@ +package nix + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + apperrors "gitlab.com/technofab/nixtest/internal/errors" +) + +func TestHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + + args := os.Args + for len(args) > 0 && args[0] != "--" { + args = args[1:] + } + if len(args) == 0 { + fmt.Fprintln(os.Stderr, "No command after --") + os.Exit(1) + } + args = args[1:] + + cmd, params := args[0], args[1:] + + switch cmd { + case "nix": + if len(params) > 0 && params[0] == "build" { + mockOutput := os.Getenv("MOCK_NIX_BUILD_OUTPUT") + mockError := os.Getenv("MOCK_NIX_BUILD_ERROR") + mockExitCode := os.Getenv("MOCK_NIX_BUILD_EXIT_CODE") + + if mockError != "" { + fmt.Fprintln(os.Stderr, mockError) + } + if mockExitCode != "" && mockExitCode != "0" { + os.Exit(1) // simplified exit for helper + } + if mockError == "" && (mockExitCode == "" || mockExitCode == "0") { + fmt.Fprintln(os.Stdout, mockOutput) + } + } + case "bash", "env": + scriptPath := params[0] + if cmd == "env" && len(params) > 2 { + scriptPath = params[2] + } + if _, err := os.Stat(scriptPath); err != nil && !os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "mocked script: script path %s could not be statted: %v\n", scriptPath, err) + os.Exit(3) + } + fmt.Fprint(os.Stdout, os.Getenv("MOCK_SCRIPT_STDOUT")) + fmt.Fprint(os.Stderr, os.Getenv("MOCK_SCRIPT_STDERR")) + if code := os.Getenv("MOCK_SCRIPT_EXIT_CODE"); code != "" && code != "0" { + os.Exit(5) // custom exit for script failure + } + default: + fmt.Fprintf(os.Stderr, "mocked command: unknown command %s\n", cmd) + os.Exit(126) + } +} + +// mockExecCommand configures the DefaultService to use the test helper +func mockExecCommandForService(service *DefaultService) { + service.commandExecutor = func(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + cmd.Env = append(cmd.Env, os.Environ()...) + return cmd + } +} + +func TestDefaultService_BuildDerivation(t *testing.T) { + service := NewDefaultService() + mockExecCommandForService(service) // configure service to use helper + + tests := []struct { + name string + derivation string + mockOutput string + mockError string + mockExitCode string + wantPath string + wantErr bool + wantErrType any + wantErrMsgContains string + }{ + {"Success", "some.drv#attr", "/nix/store/mock-path", "", "0", "/nix/store/mock-path", false, nil, ""}, + { + "Nix command error", "error.drv#attr", "", "nix error details", "1", "", true, + (*apperrors.NixBuildError)(nil), "nix error details", + }, + { + "Nix command success but no output path", "empty.drv#attr", "", "", "0", "", true, + (*apperrors.NixNoOutputPathError)(nil), "produced no output path", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv("MOCK_NIX_BUILD_OUTPUT", tt.mockOutput) + os.Setenv("MOCK_NIX_BUILD_ERROR", tt.mockError) + os.Setenv("MOCK_NIX_BUILD_EXIT_CODE", tt.mockExitCode) + defer func() { + os.Unsetenv("MOCK_NIX_BUILD_OUTPUT") + os.Unsetenv("MOCK_NIX_BUILD_ERROR") + os.Unsetenv("MOCK_NIX_BUILD_EXIT_CODE") + }() + + gotPath, err := service.BuildDerivation(tt.derivation) + + if (err != nil) != tt.wantErr { + t.Fatalf("BuildDerivation() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + if tt.wantErrType != nil && !errors.As(err, &tt.wantErrType) { + t.Errorf("BuildDerivation() error type = %T, want %T", err, tt.wantErrType) + } + if tt.wantErrMsgContains != "" && !strings.Contains(err.Error(), tt.wantErrMsgContains) { + t.Errorf("BuildDerivation() error = %q, want error containing %q", err.Error(), tt.wantErrMsgContains) + } + } + if !tt.wantErr && gotPath != tt.wantPath { + t.Errorf("BuildDerivation() gotPath = %v, want %v", gotPath, tt.wantPath) + } + }) + } +} + +func TestDefaultService_BuildAndParseJSON(t *testing.T) { + service := NewDefaultService() + mockExecCommandForService(service) + + tempDir := t.TempDir() + mockDrvOutputPath := filepath.Join(tempDir, "drv_output.json") + + tests := []struct { + name string + derivation string + mockBuildOutput string + mockJSONContent string + mockBuildError string + mockBuildExitCode string + want any + wantErr bool + wantErrType any + wantErrMsgContains string + }{ + { + "Success", "some.drv#json", mockDrvOutputPath, `{"key": "value"}`, "", "0", + map[string]any{"key": "value"}, false, nil, "", + }, + { + "BuildDerivation fails", "error.drv#json", "", "", "nix build error", "1", + nil, true, (*apperrors.NixBuildError)(nil), "nix build error", + }, + { + "ReadFile fails", "readfail.drv#json", "/nonexistent/path/output.json", "", "", "0", + nil, true, (*apperrors.FileReadError)(nil), "failed to read file", + }, + { + "Unmarshal fails", "badjson.drv#json", mockDrvOutputPath, `{"key": "value"`, "", "0", + nil, true, (*apperrors.JSONUnmarshalError)(nil), "failed to unmarshal JSON", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv("MOCK_NIX_BUILD_OUTPUT", tt.mockBuildOutput) + os.Setenv("MOCK_NIX_BUILD_ERROR", tt.mockBuildError) + os.Setenv("MOCK_NIX_BUILD_EXIT_CODE", tt.mockBuildExitCode) + + if tt.mockJSONContent != "" && tt.mockBuildOutput == mockDrvOutputPath { + if err := os.WriteFile(mockDrvOutputPath, []byte(tt.mockJSONContent), 0644); err != nil { + t.Fatalf("Failed to write mock JSON content: %v", err) + } + defer os.Remove(mockDrvOutputPath) + } + + got, err := service.BuildAndParseJSON(tt.derivation) + + if (err != nil) != tt.wantErr { + t.Fatalf("BuildAndParseJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + if tt.wantErrType != nil && !errors.As(err, &tt.wantErrType) { + t.Errorf("BuildAndParseJSON() error type = %T, want %T", err, tt.wantErrType) + } + if tt.wantErrMsgContains != "" && !strings.Contains(err.Error(), tt.wantErrMsgContains) { + t.Errorf("BuildAndParseJSON() error = %q, want error containing %q", err.Error(), tt.wantErrMsgContains) + } + } + if !tt.wantErr && !jsonDeepEqual(got, tt.want) { + t.Errorf("BuildAndParseJSON() got = %v, want %v", got, tt.want) + } + }) + } +} + +func jsonDeepEqual(a, b any) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + jsonA, _ := json.Marshal(a) + jsonB, _ := json.Marshal(b) + return string(jsonA) == string(jsonB) +} + +func TestDefaultService_BuildAndRunScript(t *testing.T) { + service := NewDefaultService() + mockExecCommandForService(service) + + tempDir := t.TempDir() + mockScriptPath := filepath.Join(tempDir, "mock_script.sh") + if err := os.WriteFile(mockScriptPath, []byte("#!/bin/bash\necho hello"), 0755); err != nil { + t.Fatalf("Failed to create dummy mock script: %v", err) + } + + tests := []struct { + name string + derivation string + pureEnv bool + mockBuildDrvOutput string + mockBuildDrvError string + mockBuildDrvExitCode string + mockScriptStdout string + mockScriptStderr string + mockScriptExitCode string + wantExitCode int + wantStdout string + wantStderr string + wantErr bool + wantErrType any + wantErrMsgContains string + }{ + { + "Success", "script.drv#sh", false, mockScriptPath, "", "0", + "Hello", "ErrOut", "0", + 0, "Hello", "ErrOut", false, nil, "", + }, + { + "Success pure", "script.drv#sh", true, mockScriptPath, "", "0", + "Hello", "ErrOut", "0", + 0, "Hello", "ErrOut", false, nil, "", + }, + { + "Script fails (non-zero exit)", "fail.drv#sh", false, mockScriptPath, "", "0", + "Out", "Err", "custom", // helper uses 5 for script failure + 5, "Out", "Err", false, nil, "", // error is nil, non-zero exit code + }, + { + "BuildDerivation fails", "buildfail.drv#sh", false, "", "nix error", "1", + "", "", "", + -1, "", "", true, (*apperrors.NixBuildError)(nil), "nix error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + os.Setenv("MOCK_NIX_BUILD_OUTPUT", tt.mockBuildDrvOutput) + os.Setenv("MOCK_NIX_BUILD_ERROR", tt.mockBuildDrvError) + os.Setenv("MOCK_NIX_BUILD_EXIT_CODE", tt.mockBuildDrvExitCode) + os.Setenv("MOCK_SCRIPT_STDOUT", tt.mockScriptStdout) + os.Setenv("MOCK_SCRIPT_STDERR", tt.mockScriptStderr) + os.Setenv("MOCK_SCRIPT_EXIT_CODE", tt.mockScriptExitCode) + + exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.pureEnv) + + if (err != nil) != tt.wantErr { + t.Fatalf("BuildAndRunScript() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + if tt.wantErrType != nil && !errors.As(err, &tt.wantErrType) { + t.Errorf("BuildAndRunScript() error type = %T, want %T", err, tt.wantErrType) + } + if tt.wantErrMsgContains != "" && !strings.Contains(err.Error(), tt.wantErrMsgContains) { + t.Errorf("BuildAndRunScript() error = %q, want error containing %q", err.Error(), tt.wantErrMsgContains) + } + } else { + if exitCode != tt.wantExitCode { + t.Errorf("BuildAndRunScript() exitCode = %v, want %v", exitCode, tt.wantExitCode) + } + if stdout != tt.wantStdout { + t.Errorf("BuildAndRunScript() stdout = %q, want %q", stdout, tt.wantStdout) + } + if stderr != tt.wantStderr { + t.Errorf("BuildAndRunScript() stderr = %q, want %q", stderr, tt.wantStderr) + } + } + }) + } +} diff --git a/internal/report/console/console.go b/internal/report/console/console.go new file mode 100644 index 0000000..959741c --- /dev/null +++ b/internal/report/console/console.go @@ -0,0 +1,143 @@ +package console + +import ( + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" + "github.com/rs/zerolog/log" + "github.com/sergi/go-diff/diffmatchpatch" + "gitlab.com/technofab/nixtest/internal/types" + "gitlab.com/technofab/nixtest/internal/util" +) + +// PrintErrors prints error messages for failed tests +func PrintErrors(results types.Results, noColor bool) { + for _, suiteResults := range results { + for _, result := range suiteResults { + if result.Status == types.StatusSuccess || result.Status == types.StatusSkipped { + continue + } + fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s/%s\" failed:", result.Spec.Suite, result.Spec.Name)) + message := result.ErrorMessage + if result.Status == types.StatusFailure && message == "" { + if noColor { + var err error + message, err = util.ComputeDiff(result.Expected, result.Actual) + if err != nil { + log.Panic().Err(err).Msg("failed to compute diff") + } + } else { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(result.Expected, result.Actual, true) + message = fmt.Sprintf("Diff:\n%s", dmp.DiffPrettyText(diffs)) + } + } + + if message == "" { + message = "- no output -" + } + + for _, line := range strings.Split(strings.TrimRight(message, "\n"), "\n") { + fmt.Printf("%s %s\n", text.FgRed.Sprint("|"), line) + } + fmt.Println() + } + } +} + +// PrintSummary prints a table summarizing test results +func PrintSummary(results types.Results, totalSuccessCount int, totalTestCount int) { + t := table.NewWriter() + t.SetStyle(table.StyleLight) + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{"Suite / Test", "Duration", "Status", "File:Line"}) + + log.Info().Msg("Summary:") + + suiteNames := make([]string, 0, len(results)) + for name := range results { + suiteNames = append(suiteNames, name) + } + sort.Strings(suiteNames) + + for _, suiteName := range suiteNames { + suiteResults := results[suiteName] + suiteTotal := len(suiteResults) + suiteSuccess := 0 + suiteSkipped := 0 + + for _, res := range suiteResults { + if res.Status == types.StatusSuccess { + suiteSuccess++ + } else if res.Status == types.StatusSkipped { + suiteSkipped++ + } + } + + statusStr := fmt.Sprintf("%d/%d", suiteSuccess, suiteTotal) + if suiteSkipped > 0 { + statusStr += fmt.Sprintf(" (%d skipped)", suiteSkipped) + } + + t.AppendRow(table.Row{ + text.Bold.Sprint(suiteName), + "", + statusStr, + "", + }) + + sort.Slice(suiteResults, func(i, j int) bool { + return suiteResults[i].Spec.Name < suiteResults[j].Spec.Name + }) + + for _, res := range suiteResults { + var symbol string + switch res.Status { + case types.StatusSuccess: + symbol = text.FgGreen.Sprint("✅ PASS") + case types.StatusFailure: + symbol = text.FgRed.Sprint("❌ FAIL") + case types.StatusError: + symbol = text.FgYellow.Sprint("❗ ERROR") + case types.StatusSkipped: + symbol = text.FgBlue.Sprint("⏭️ SKIP") + default: + symbol = "UNKNOWN" + } + + t.AppendRow([]any{ + " " + res.Spec.Name, + fmt.Sprintf("%s", res.Duration.Round(time.Millisecond)), + symbol, + res.Spec.Pos, + }) + } + t.AppendSeparator() + } + + overallStatusStr := fmt.Sprintf("%d/%d", totalSuccessCount, totalTestCount) + totalSkipped := 0 + for _, suiteResults := range results { + for _, res := range suiteResults { + if res.Status == types.StatusSkipped { + totalSkipped++ + } + } + } + if totalSkipped > 0 { + overallStatusStr += fmt.Sprintf(" (%d skipped)", totalSkipped) + } + + t.AppendFooter(table.Row{ + text.Bold.Sprint("TOTAL"), + "", + text.Bold.Sprint(overallStatusStr), + "", + }) + t.Render() +} diff --git a/internal/report/console/console_test.go b/internal/report/console/console_test.go new file mode 100644 index 0000000..bbe9a6a --- /dev/null +++ b/internal/report/console/console_test.go @@ -0,0 +1,228 @@ +package console + +import ( + "bytes" + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/jedib0t/go-pretty/v6/text" + "gitlab.com/technofab/nixtest/internal/types" +) + +// captureOutput captures stdout and stderr during the execution of a function +func captureOutput(f func()) (string, string) { + // save original stdout and stderr + originalStdout := os.Stdout + originalStderr := os.Stderr + + rOut, wOut, _ := os.Pipe() + rErr, wErr, _ := os.Pipe() + + os.Stdout = wOut + os.Stderr = wErr + + defer func() { + // restore stdout & stderr + os.Stdout = originalStdout + os.Stderr = originalStderr + }() + + outC := make(chan string) + errC := make(chan string) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + var buf bytes.Buffer + wg.Done() + _, _ = io.Copy(&buf, rOut) + outC <- buf.String() + }() + + wg.Add(1) + go func() { + var buf bytes.Buffer + wg.Done() + _, _ = io.Copy(&buf, rErr) + errC <- buf.String() + }() + + f() + + wOut.Close() + wErr.Close() + wg.Wait() + + stdout := <-outC + stderr := <-errC + + return stdout, stderr +} + +func TestPrintErrorsColor(t *testing.T) { + results := types.Results{ + "Suite1": []types.TestResult{ + { + Spec: types.TestSpec{Suite: "Suite1", Name: "TestFailure_Diff"}, + Status: types.StatusFailure, + Expected: "line1\nline2 expected\nline3", + Actual: "line1\nline2 actual\nline3", + }, + }, + } + stdout, _ := captureOutput(func() { + PrintErrors(results, false) + }) + + ansiEscapePattern := `(?:\\x1b\[[0-9;]*m)*` + pattern := `.*\n` + + `A|A Diff:\n` + + `A|A line1\n` + + `A|A line2 AexpeAAacAAtedAAualA\n` + + `A|A line3.*` + pattern = strings.ReplaceAll(pattern, "A", ansiEscapePattern) + + matched, _ := regexp.MatchString(pattern, stdout) + + if !matched { + t.Errorf("PrintErrors() TestFailure_Diff diff output mismatch or missing.\nExpected pattern:\n%s\nGot:\n%s", pattern, stdout) + } +} + +func TestPrintErrors(t *testing.T) { + text.DisableColors() + defer text.EnableColors() + + results := types.Results{ + "Suite1": []types.TestResult{ + { + Spec: types.TestSpec{Suite: "Suite1", Name: "TestSuccess"}, + Status: types.StatusSuccess, + }, + { + Spec: types.TestSpec{Suite: "Suite1", Name: "TestFailure_Diff"}, + Status: types.StatusFailure, + Expected: "line1\nline2 expected\nline3", + Actual: "line1\nline2 actual\nline3", + }, + { + Spec: types.TestSpec{Suite: "Suite1", Name: "TestFailure_Message"}, + Status: types.StatusFailure, + ErrorMessage: "This is a specific failure message.\nWith multiple lines.", + }, + { + Spec: types.TestSpec{Suite: "Suite1", Name: "TestError"}, + Status: types.StatusError, + ErrorMessage: "System error occurred.", + }, + { + Spec: types.TestSpec{Suite: "Suite1", Name: "TestEmpty"}, + Status: types.StatusError, + ErrorMessage: "", + }, + }, + } + + stdout, _ := captureOutput(func() { + PrintErrors(results, true) + }) + + if strings.Contains(stdout, "TestSuccess") { + t.Errorf("PrintErrors() should not print success cases, but found 'TestSuccess'") + } + + expectedDiffPattern := `\|\s*--- expected\s*\n` + // matches "| --- expected" + `\|\s*\+\+\+ actual\s*\n` + // matches "| +++ actual" + `\|\s*@@ -\d+,\d+ \+\d+,\d+ @@\s*\n` + // matches "| @@ @@" + `\|\s* line1\s*\n` + // matches "| line1" (note the leading space for an "equal" line) + `\|\s*-line2 expected\s*\n` + // matches "| -line2 expected" + `\|\s*\+line2 actual\s*\n` + // matches "| +line2 actual" + `\|\s* line3\s*` // matches "| line3" + + matched, _ := regexp.MatchString(expectedDiffPattern, stdout) + + if !matched { + t.Errorf("PrintErrors() TestFailure_Diff diff output mismatch or missing.\nExpected pattern:\n%s\nGot:\n%s", expectedDiffPattern, stdout) + } + + if !strings.Contains(stdout, "⚠ Test \"Suite1/TestFailure_Message\" failed:") { + t.Errorf("PrintErrors() missing header for TestFailure_Message. Output:\n%s", stdout) + } + + if !strings.Contains(stdout, "| This is a specific failure message.") || + + !strings.Contains(stdout, "| With multiple lines.") { + t.Errorf("PrintErrors() TestFailure_Message message output mismatch or missing. Output:\n%s", stdout) + } + + if !strings.Contains(stdout, "⚠ Test \"Suite1/TestError\" failed:") { + t.Errorf("PrintErrors() missing header for TestError. Output:\n%s", stdout) + } + + if !strings.Contains(stdout, "| System error occurred.") { + t.Errorf("PrintErrors() TestError message output mismatch or missing. Output:\n%s", stdout) + } + if !strings.Contains(stdout, "- no output -") { + t.Errorf("PrintErrors() missing '- no output -'. Output:\n%s", stdout) + } +} + +func TestPrintSummary(t *testing.T) { + text.DisableColors() + defer text.EnableColors() + + results := types.Results{ + "AlphaSuite": []types.TestResult{ + {Spec: types.TestSpec{Suite: "AlphaSuite", Name: "TestA", Pos: "alpha.nix:1"}, Status: types.StatusSuccess, Duration: 100 * time.Millisecond}, + {Spec: types.TestSpec{Suite: "AlphaSuite", Name: "TestB", Pos: "alpha.nix:2"}, Status: types.StatusFailure, Duration: 200 * time.Millisecond}, + }, + "BetaSuite": []types.TestResult{ + {Spec: types.TestSpec{Suite: "BetaSuite", Name: "TestC", Pos: "beta.nix:1"}, Status: types.StatusSkipped, Duration: 50 * time.Millisecond}, + {Spec: types.TestSpec{Suite: "BetaSuite", Name: "TestD", Pos: "beta.nix:2"}, Status: types.StatusError, Duration: 150 * time.Millisecond}, + {Spec: types.TestSpec{Suite: "BetaSuite", Name: "TestE", Pos: "beta.nix:3"}, Status: 123, Duration: 150 * time.Millisecond}, + }, + } + totalSuccessCount := 2 + totalTestCount := 4 + + r, w, _ := os.Pipe() + originalStdout := os.Stdout + os.Stdout = w + + PrintSummary(results, totalSuccessCount, totalTestCount) + + w.Close() + os.Stdout = originalStdout + + var summaryTable bytes.Buffer + _, _ = io.Copy(&summaryTable, r) + stdout := summaryTable.String() + + if !strings.Contains(stdout, "AlphaSuite") || !strings.Contains(stdout, "BetaSuite") { + t.Errorf("PrintSummary() missing suite names. Output:\n%s", stdout) + } + + // check for test names and statuses + if !strings.Contains(stdout, "TestA") || !strings.Contains(stdout, "PASS") { + t.Errorf("PrintSummary() missing TestA or its PASS status. Output:\n%s", stdout) + } + if !strings.Contains(stdout, "TestB") || !strings.Contains(stdout, "FAIL") { + t.Errorf("PrintSummary() missing TestB or its FAIL status. Output:\n%s", stdout) + } + if !strings.Contains(stdout, "TestE") || !strings.Contains(stdout, "UNKNOWN") { + t.Errorf("PrintSummary() missing TestE or its UNKNOWN status. Output:\n%s", stdout) + } + + // check for total summary + expectedTotalSummary := fmt.Sprintf("%d/%d (1 SKIPPED)", totalSuccessCount, totalTestCount) + if !strings.Contains(stdout, expectedTotalSummary) { + t.Errorf("PrintSummary() total summary incorrect. Expected to contain '%s'. Output:\n%s", expectedTotalSummary, stdout) + } +} diff --git a/internal/report/junit/junit.go b/internal/report/junit/junit.go new file mode 100644 index 0000000..a376600 --- /dev/null +++ b/internal/report/junit/junit.go @@ -0,0 +1,158 @@ +package junit + +import ( + "encoding/xml" + "fmt" + "os" + "strings" + "time" + + "gitlab.com/technofab/nixtest/internal/types" + "gitlab.com/technofab/nixtest/internal/util" +) + +type JUnitReport struct { + XMLName xml.Name `xml:"testsuites"` + Name string `xml:"name,attr,omitempty"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` + Time string `xml:"time,attr"` + Suites []JUnitTestSuite `xml:"testsuite"` +} + +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` + Time string `xml:"time,attr"` + TestCases []JUnitCase `xml:"testcase"` +} + +type JUnitCase struct { + XMLName xml.Name `xml:"testcase"` + Name string `xml:"name,attr"` + Classname string `xml:"classname,attr"` + Time string `xml:"time,attr"` + File string `xml:"file,attr,omitempty"` + Line string `xml:"line,attr,omitempty"` + Failure *JUnitFailure `xml:"failure,omitempty"` + Error *JUnitError `xml:"error,omitempty"` + Skipped *JUnitSkipped `xml:"skipped,omitempty"` +} + +type JUnitFailure struct { + XMLName xml.Name `xml:"failure"` + Message string `xml:"message,attr,omitempty"` + Data string `xml:",cdata"` +} + +type JUnitError struct { + XMLName xml.Name `xml:"error"` + Message string `xml:"message,attr,omitempty"` + Data string `xml:",cdata"` +} + +type JUnitSkipped struct { + XMLName xml.Name `xml:"skipped"` + Message string `xml:"message,attr,omitempty"` +} + +// GenerateReport generates the Junit XML content as a string +func GenerateReport(reportName string, results types.Results) (string, error) { + report := JUnitReport{ + Name: reportName, + Suites: []JUnitTestSuite{}, + } + totalDuration := time.Duration(0) + + for suiteName, suiteResults := range results { + suite := JUnitTestSuite{ + Name: suiteName, + Tests: len(suiteResults), + TestCases: []JUnitCase{}, + } + suiteDuration := time.Duration(0) + + for _, result := range suiteResults { + durationSeconds := fmt.Sprintf("%.3f", result.Duration.Seconds()) + totalDuration += result.Duration + suiteDuration += result.Duration + + testCase := JUnitCase{ + Name: result.Spec.Name, + Classname: suiteName, + Time: durationSeconds, + } + + if result.Spec.Pos != "" { + parts := strings.SplitN(result.Spec.Pos, ":", 2) + testCase.File = parts[0] + if len(parts) > 1 { + testCase.Line = parts[1] + } + } + + switch result.Status { + case types.StatusFailure: + suite.Failures++ + report.Failures++ + var failureContent string + if result.ErrorMessage != "" { + failureContent = result.ErrorMessage + } else { + var err error + failureContent, err = util.ComputeDiff(result.Expected, result.Actual) + if err != nil { + return "", fmt.Errorf("failed to compute diff") + } + } + testCase.Failure = &JUnitFailure{Message: "Test failed", Data: failureContent} + case types.StatusError: + suite.Errors++ + report.Errors++ + testCase.Error = &JUnitError{Message: "Test errored", Data: result.ErrorMessage} + case types.StatusSkipped: + suite.Skipped++ + report.Skipped++ + testCase.Skipped = &JUnitSkipped{Message: "Test skipped"} + } + report.Tests++ + suite.TestCases = append(suite.TestCases, testCase) + } + suite.Time = fmt.Sprintf("%.3f", suiteDuration.Seconds()) + report.Suites = append(report.Suites, suite) + } + + report.Time = fmt.Sprintf("%.3f", totalDuration.Seconds()) + + output, err := xml.MarshalIndent(report, "", " ") + if err != nil { + return "", fmt.Errorf("failed to marshal XML: %w", err) + } + return xml.Header + string(output), nil +} + +// WriteFile generates a Junit report and writes it to the specified path +func WriteFile(filePath string, reportName string, results types.Results) error { + xmlContent, err := GenerateReport(reportName, results) + if err != nil { + return fmt.Errorf("failed to generate junit report content: %w", err) + } + + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create junit file %s: %w", filePath, err) + } + defer file.Close() + + _, err = file.WriteString(xmlContent) + if err != nil { + return fmt.Errorf("failed to write junit report to %s: %w", filePath, err) + } + return nil +} diff --git a/internal/report/junit/junit_test.go b/internal/report/junit/junit_test.go new file mode 100644 index 0000000..fbca0aa --- /dev/null +++ b/internal/report/junit/junit_test.go @@ -0,0 +1,116 @@ +package junit + +import ( + "encoding/xml" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitlab.com/technofab/nixtest/internal/types" +) + +func formatDurationSeconds(d time.Duration) string { + return fmt.Sprintf("%.3f", d.Seconds()) +} + +func TestGenerateReport(t *testing.T) { + results := types.Results{ + "Suite1": []types.TestResult{ + { + Spec: types.TestSpec{Name: "Test1_Success", Suite: "Suite1", Pos: "file1.nix:10"}, + Status: types.StatusSuccess, + Duration: 123 * time.Millisecond, + }, + { + Spec: types.TestSpec{Name: "Test2_Failure", Suite: "Suite1", Pos: "file1.nix:20"}, + Status: types.StatusFailure, + Duration: 234 * time.Millisecond, + Expected: "hello", + Actual: "world", + }, + }, + "Suite2": []types.TestResult{ + { + Spec: types.TestSpec{Name: "Test3_Error", Suite: "Suite2"}, + Status: types.StatusError, + Duration: 345 * time.Millisecond, + ErrorMessage: "Something went very wrong", + }, + { + Spec: types.TestSpec{Name: "Test4_Failure_Message", Suite: "Suite2"}, + Status: types.StatusFailure, + Duration: 456 * time.Millisecond, + ErrorMessage: "hello world", + }, + { + Spec: types.TestSpec{Name: "Test5_Skipped", Suite: "Suite2"}, + Status: types.StatusSkipped, + Duration: 567 * time.Millisecond, + }, + }, + } + + totalDuration := (123 + 234 + 345 + 456 + 567) * time.Millisecond + reportName := "MyNixtestReport" + xmlString, err := GenerateReport(reportName, results) + if err != nil { + t.Fatalf("GenerateReport() failed: %v", err) + } + + if !strings.HasPrefix(xmlString, xml.Header) { + t.Error("GenerateReport() output missing XML header") + } + if !strings.Contains(xmlString, ". Got: %s", xmlString) + } + if !strings.Contains(xmlString, "tests=\"5\"") { + t.Errorf("GenerateReport() incorrect total tests count. Got: %s", xmlString) + } + if !strings.Contains(xmlString, "failures=\"2\"") { + t.Errorf("GenerateReport() incorrect total failures count. Got: %s", xmlString) + } + if !strings.Contains(xmlString, "errors=\"1\"") { + t.Errorf("GenerateReport() incorrect total errors count. Got: %s", xmlString) + } + if !strings.Contains(xmlString, "time=\""+formatDurationSeconds(totalDuration)+"\"") { + t.Errorf("GenerateReport() incorrect total time. Expected %s. Got part: %s", formatDurationSeconds(totalDuration), xmlString) + } + + var report JUnitReport + if err := xml.Unmarshal([]byte(strings.TrimPrefix(xmlString, xml.Header)), &report); err != nil { + t.Fatalf("Failed to unmarshal generated XML: %v\nXML:\n%s", err, xmlString) + } + + if report.Name != reportName { + t.Errorf("Report.Name = %q, want %q", report.Name, reportName) + } + if len(report.Suites) != 2 { + t.Fatalf("Report.Suites length = %d, want 2", len(report.Suites)) + } +} + +func TestWriteFile(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "junit_report.xml") + results := types.Results{ + "SuiteSimple": []types.TestResult{ + {Spec: types.TestSpec{Name: "SimpleTest", Suite: "SuiteSimple"}, Status: types.StatusSuccess, Duration: 1 * time.Second}, + }, + } + + err := WriteFile(filePath, "TestReport", results) + if err != nil { + t.Fatalf("WriteFile() failed: %v", err) + } + + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read written JUnit file: %v", err) + } + if !strings.Contains(string(content), " main test execution logic +func (r *Runner) executeTest(spec types.TestSpec) types.TestResult { + startTime := time.Now() + result := types.TestResult{ + Spec: spec, + Status: types.StatusSuccess, + } + + if r.shouldSkip(spec.Name) { + result.Status = types.StatusSkipped + result.Duration = time.Since(startTime) + return result + } + + var actual any + var err error + + if spec.ActualDrv != "" { + actual, err = r.nixService.BuildAndParseJSON(spec.ActualDrv) + if err != nil { + result.Status = types.StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to build/parse actualDrv %s: %v", spec.ActualDrv, err) + goto end + } + } else { + actual = spec.Actual + } + + switch spec.Type { + case types.TestTypeSnapshot: + r.handleSnapshotTest(&result, spec, actual) + case types.TestTypeUnit: + r.handleUnitTest(&result, spec, actual) + case types.TestTypeScript: + r.handleScriptTest(&result, spec) + default: + result.Status = types.StatusError + result.ErrorMessage = fmt.Sprintf("Invalid test type: %s", spec.Type) + } + +end: + result.Duration = time.Since(startTime) + return result +} + +// handleSnapshotTest processes snapshot type tests +func (r *Runner) handleSnapshotTest(result *types.TestResult, spec types.TestSpec, actual any) { + snapPath := r.snapService.GetPath(r.config.SnapshotDir, spec.Name) + + if r.config.UpdateSnapshots { + if err := r.snapService.CreateFile(snapPath, actual); err != nil { + result.Status = types.StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to update snapshot %s: %v", snapPath, err) + return + } + log.Info().Str("test", spec.Name).Str("path", snapPath).Msg("Snapshot updated") + } + + _, statErr := r.snapService.Stat(snapPath) + if statErr != nil { + result.ErrorMessage = fmt.Sprintf("[system] failed to stat snapshot %s: %v", snapPath, statErr) + result.Status = types.StatusError + return + } + + expected, err := r.snapService.LoadFile(snapPath) + if err != nil { + result.Status = types.StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to parse snapshot %s: %v", snapPath, err) + return + } + + r.compareActualExpected(result, actual, expected) +} + +// handleUnitTest processes unit type tests +func (r *Runner) handleUnitTest(result *types.TestResult, spec types.TestSpec, actual any) { + expected := spec.Expected + r.compareActualExpected(result, actual, expected) +} + +// handleScriptTest processes script type tests +func (r *Runner) handleScriptTest(result *types.TestResult, spec types.TestSpec) { + exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.PureEnv) + if err != nil { + result.Status = types.StatusError + result.ErrorMessage = fmt.Sprintf("[system] failed to run script derivation %s: %v", spec.Script, err) + return + } + if exitCode != 0 { + result.Status = types.StatusFailure + result.ErrorMessage = fmt.Sprintf("[exit code %d]\n[stdout]\n%s\n[stderr]\n%s", exitCode, stdout, stderrStr) + } +} + +// compareActualExpected performs the deep equality check and formats diffs +func (r *Runner) compareActualExpected(result *types.TestResult, actual, expected any) { + if reflect.DeepEqual(actual, expected) { + // if we already have an error don't overwrite it + if result.Status != types.StatusError { + result.Status = types.StatusSuccess + } + } else { + result.Status = types.StatusFailure + + var actualStr, expectedStr string + var marshalErr error + + if util.IsString(actual) && util.IsString(expected) { + actualStr = actual.(string) + expectedStr = expected.(string) + } else { + expectedBytes, err := json.MarshalIndent(expected, "", " ") + if err != nil { + marshalErr = fmt.Errorf("[system] failed to marshal 'expected' for diff: %w", err) + } else { + expectedStr = string(expectedBytes) + } + + actualBytes, err := json.MarshalIndent(actual, "", " ") + if err != nil && marshalErr == nil { + marshalErr = fmt.Errorf("[system] failed to marshal 'actual' for diff: %w", err) + } else if err == nil && marshalErr == nil { + actualStr = string(actualBytes) + } + } + + if marshalErr != nil { + result.Status = types.StatusError + result.ErrorMessage = marshalErr.Error() + return + } + + result.Expected = expectedStr + result.Actual = actualStr + } +} diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go new file mode 100644 index 0000000..1415dd8 --- /dev/null +++ b/internal/runner/runner_test.go @@ -0,0 +1,390 @@ +package runner + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + apperrors "gitlab.com/technofab/nixtest/internal/errors" + "gitlab.com/technofab/nixtest/internal/types" +) + +// --- Mock Service Implementations --- + +type mockNixService struct { + BuildDerivationFunc func(derivation string) (string, error) + BuildAndParseJSONFunc func(derivation string) (any, error) + BuildAndRunScriptFunc func(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) +} + +func (m *mockNixService) BuildDerivation(d string) (string, error) { + if m.BuildDerivationFunc == nil { + panic("mockNixService.BuildDerivationFunc not set") + } + return m.BuildDerivationFunc(d) +} +func (m *mockNixService) BuildAndParseJSON(d string) (any, error) { + if m.BuildAndParseJSONFunc == nil { + panic("mockNixService.BuildAndParseJSONFunc not set") + } + return m.BuildAndParseJSONFunc(d) +} +func (m *mockNixService) BuildAndRunScript(d string, p bool) (int, string, string, error) { + if m.BuildAndRunScriptFunc == nil { + panic("mockNixService.BuildAndRunScriptFunc not set") + } + return m.BuildAndRunScriptFunc(d, p) +} + +type mockSnapshotService struct { + GetPathFunc func(snapshotDir string, testName string) string + CreateFileFunc func(filePath string, data any) error + LoadFileFunc func(filePath string) (any, error) + StatFunc func(name string) (os.FileInfo, error) +} + +func (m *mockSnapshotService) GetPath(sDir string, tName string) string { + if m.GetPathFunc == nil { // provide a default if not overridden + return filepath.Join(sDir, strings.ToLower(strings.ReplaceAll(tName, " ", "_"))+".snap.json") + } + return m.GetPathFunc(sDir, tName) +} +func (m *mockSnapshotService) CreateFile(fp string, d any) error { + if m.CreateFileFunc == nil { + panic("mockSnapshotService.CreateFileFunc not set") + } + return m.CreateFileFunc(fp, d) +} +func (m *mockSnapshotService) LoadFile(fp string) (any, error) { + if m.LoadFileFunc == nil { + panic("mockSnapshotService.LoadFileFunc not set") + } + return m.LoadFileFunc(fp) +} +func (m *mockSnapshotService) Stat(n string) (os.FileInfo, error) { + if m.StatFunc == nil { + panic("mockSnapshotService.StatFunc not set") + } + return m.StatFunc(n) +} + +// mockFileInfo for snapshot.Stat +type mockFileInfo struct { + name string + isDir bool + modTime time.Time + size int64 + mode os.FileMode +} + +func (m mockFileInfo) Name() string { return m.name } +func (m mockFileInfo) IsDir() bool { return m.isDir } +func (m mockFileInfo) ModTime() time.Time { return m.modTime } +func (m mockFileInfo) Size() int64 { return m.size } +func (m mockFileInfo) Mode() os.FileMode { return m.mode } +func (m mockFileInfo) Sys() any { return nil } + +// --- Test Cases --- + +func TestNewRunner(t *testing.T) { + mockNix := &mockNixService{} + mockSnap := &mockSnapshotService{} + tests := []struct { + name string + cfg Config + wantErr bool + skipPattern string + }{ + {"Valid config, no skip", Config{NumWorkers: 1}, false, ""}, + {"Valid config, valid skip", Config{NumWorkers: 1, SkipPattern: "Test.*"}, false, "Test.*"}, + {"Invalid skip pattern", Config{NumWorkers: 1, SkipPattern: "[invalid"}, true, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, err := New(tt.cfg, mockNix, mockSnap) + if (err != nil) != tt.wantErr { + t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err == nil { + if tt.skipPattern == "" && r.skipRegex != nil { + t.Errorf("Expected nil skipRegex, got %v", r.skipRegex) + } + if tt.skipPattern != "" && (r.skipRegex == nil || r.skipRegex.String() != tt.skipPattern) { + t.Errorf("Expected skipRegex %q, got %v", tt.skipPattern, r.skipRegex) + } + } + }) + } +} + +func TestRunner_executeTest(t *testing.T) { + tempDir := t.TempDir() // used for snapshotDir in runnerConfig + + tests := []struct { + name string + spec types.TestSpec + runnerConfig Config + setupMockServices func(t *testing.T, mockNix *mockNixService, mockSnap *mockSnapshotService, spec types.TestSpec, cfg Config) + wantStatus types.TestStatus + wantErrMsgContains string + wantActual string + wantExpected string + }{ + // --- Invalid --- + { + name: "Invalid test type", + spec: types.TestSpec{Name: "Invalid", Type: "invalid"}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + // No service calls expected + }, + wantStatus: types.StatusError, + }, + // --- Skip --- + { + name: "Skip test due to pattern", + spec: types.TestSpec{Name: "SkipThisTest", Type: types.TestTypeUnit}, + runnerConfig: Config{SkipPattern: "SkipThis.*"}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + // No service calls expected + }, + wantStatus: types.StatusSkipped, + }, + // --- Unit Tests --- + { + name: "Unit test success", + spec: types.TestSpec{Name: "UnitSuccess", Type: types.TestTypeUnit, Expected: "hello", Actual: "hello"}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {}, + wantStatus: types.StatusSuccess, + }, + { + name: "Unit test failure", + spec: types.TestSpec{Name: "UnitFail", Type: types.TestTypeUnit, Expected: map[string]int{"a": 1}, Actual: map[string]int{"a": 2}}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {}, + wantStatus: types.StatusFailure, + wantExpected: "{\n \"a\": 1\n}", + wantActual: "{\n \"a\": 2\n}", + }, + { + name: "Unit test success with ActualDrv", + spec: types.TestSpec{Name: "UnitActualDrvSuccess", Type: types.TestTypeUnit, Expected: map[string]any{"key": "val"}, ActualDrv: "drv.actual"}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + mNix.BuildAndParseJSONFunc = func(derivation string) (any, error) { + if derivation == "drv.actual" { + return map[string]any{"key": "val"}, nil + } + return nil, fmt.Errorf("unexpected drv: %s", derivation) + } + }, + wantStatus: types.StatusSuccess, + }, + { + name: "Unit test error (ActualDrv build fail)", + spec: types.TestSpec{Name: "UnitActualDrvError", Type: types.TestTypeUnit, Expected: "any", ActualDrv: "drv.actual.fail"}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + mNix.BuildAndParseJSONFunc = func(derivation string) (any, error) { + return nil, &apperrors.NixBuildError{Derivation: "drv.actual.fail", Err: errors.New("build failed")} + } + }, + wantStatus: types.StatusError, + wantErrMsgContains: "failed to build/parse actualDrv drv.actual.fail: nix build for drv.actual.fail failed: build failed", + }, + // --- Snapshot Tests --- + { + name: "Snapshot test success (existing snapshot match)", + spec: types.TestSpec{Name: "SnapSuccess", Type: types.TestTypeSnapshot, Actual: map[string]any{"data": "match"}}, + runnerConfig: Config{SnapshotDir: tempDir}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + snapPath := mSnap.GetPath(c.SnapshotDir, s.Name) + mSnap.StatFunc = func(name string) (os.FileInfo, error) { + if name == snapPath { + return mockFileInfo{name: filepath.Base(snapPath)}, nil + } + return nil, os.ErrNotExist + } + mSnap.LoadFileFunc = func(filePath string) (any, error) { + if filePath == snapPath { + return map[string]any{"data": "match"}, nil + } + return nil, os.ErrNotExist + } + }, + wantStatus: types.StatusSuccess, + }, + { + name: "Snapshot test update (snapshot created, no prior)", + spec: types.TestSpec{Name: "SnapUpdateNew", Type: types.TestTypeSnapshot, Actual: map[string]any{"data": "new"}}, + runnerConfig: Config{SnapshotDir: tempDir, UpdateSnapshots: true}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + snapPath := mSnap.GetPath(c.SnapshotDir, s.Name) + mSnap.CreateFileFunc = func(filePath string, data any) error { + if filePath == snapPath { + return nil + } + return fmt.Errorf("unexpected create path: %s", filePath) + } + mSnap.StatFunc = func(name string) (os.FileInfo, error) { + if name == snapPath { + return mockFileInfo{name: filepath.Base(snapPath)}, nil + } + return nil, os.ErrNotExist + } + mSnap.LoadFileFunc = func(filePath string) (any, error) { + if filePath == snapPath { + return s.Actual, nil + } + return nil, os.ErrNotExist + } + }, + wantStatus: types.StatusSuccess, + }, + // --- Script Tests --- + { + name: "Script test success (exit 0)", + spec: types.TestSpec{Name: "ScriptSuccess", Type: types.TestTypeScript, Script: "script.sh"}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { + return 0, "stdout", "stderr", nil + } + }, + wantStatus: types.StatusSuccess, + }, + { + name: "Script test failure (exit non-0)", + spec: types.TestSpec{Name: "ScriptFail", Type: types.TestTypeScript, Script: "script.sh"}, + runnerConfig: Config{}, + setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { + mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { + return 1, "out on fail", "err on fail", nil + } + }, + wantStatus: types.StatusFailure, + wantErrMsgContains: "[exit code 1]\n[stdout]\nout on fail\n[stderr]\nerr on fail", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockNixSvc := &mockNixService{} + mockSnapSvc := &mockSnapshotService{} + + tt.setupMockServices(t, mockNixSvc, mockSnapSvc, tt.spec, tt.runnerConfig) + + r, err := New(tt.runnerConfig, mockNixSvc, mockSnapSvc) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + + result := r.executeTest(tt.spec) + + if result.Status != tt.wantStatus { + t.Errorf("executeTest() status = %s, want %s. ErrorMsg: %s", result.Status, tt.wantStatus, result.ErrorMessage) + } + if tt.wantErrMsgContains != "" && !strings.Contains(result.ErrorMessage, tt.wantErrMsgContains) { + t.Errorf("executeTest() ErrorMessage = %q, want to contain %q", result.ErrorMessage, tt.wantErrMsgContains) + } + if result.Status == types.StatusFailure { + if tt.wantExpected != "" && result.Expected != tt.wantExpected { + t.Errorf("executeTest() Expected diff string mismatch.\nGot:\n%s\nWant:\n%s", result.Expected, tt.wantExpected) + } + if tt.wantActual != "" && result.Actual != tt.wantActual { + t.Errorf("executeTest() Actual diff string mismatch.\nGot:\n%s\nWant:\n%s", result.Actual, tt.wantActual) + } + } + if result.Duration <= 0 && result.Status != types.StatusSkipped { + t.Errorf("executeTest() Duration = %v, want > 0", result.Duration) + } + }) + } +} + +func TestRunner_RunTests(t *testing.T) { + mockNixSvc := &mockNixService{} + mockSnapSvc := &mockSnapshotService{} + + mockNixSvc.BuildAndParseJSONFunc = func(derivation string) (any, error) { return "parsed", nil } + mockNixSvc.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { return 0, "", "", nil } + mockSnapSvc.StatFunc = func(name string) (os.FileInfo, error) { return mockFileInfo{}, nil } + mockSnapSvc.LoadFileFunc = func(filePath string) (any, error) { return "snapshot", nil } + mockSnapSvc.CreateFileFunc = func(filePath string, data any) error { return nil } + + suites := []types.SuiteSpec{ + {Name: "Suite1", Tests: []types.TestSpec{ + {Name: "S1_Test1_Pass", Type: types.TestTypeUnit, Actual: "a", Expected: "a"}, + {Name: "S1_Test2_Fail", Type: types.TestTypeUnit, Actual: "a", Expected: "b"}, + }}, + {Name: "Suite2", Tests: []types.TestSpec{ + {Name: "S2_Test1_Pass", Type: types.TestTypeUnit, Actual: "c", Expected: "c"}, + {Name: "S2_Test2_SkipThis", Type: types.TestTypeUnit, Actual: "d", Expected: "d"}, + }}, + } + + runnerCfg := Config{NumWorkers: 2, SkipPattern: ".*SkipThis.*"} + testRunner, err := New(runnerCfg, mockNixSvc, mockSnapSvc) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + + results := testRunner.RunTests(suites) + + totalTestsProcessed := 0 + suite1Results, ok1 := results["Suite1"] + if !ok1 { + t.Fatalf("Missing results for Suite1") + } + totalTestsProcessed += len(suite1Results) + + suite2Results, ok2 := results["Suite2"] + if !ok2 { + t.Fatalf("Missing results for Suite2") + } + totalTestsProcessed += len(suite2Results) + + if totalTestsProcessed != 4 { + t.Errorf("RunTests() processed %d tests, want 4", totalTestsProcessed) + } + + // Check statuses + foundS1T1, foundS1T2, foundS2T1, foundS2T2 := false, false, false, false + for _, res := range suite1Results { + if res.Spec.Name == "S1_Test1_Pass" { + foundS1T1 = true + if res.Status != types.StatusSuccess { + t.Errorf("S1_Test1_Pass status %s, want Success", res.Status) + } + } + if res.Spec.Name == "S1_Test2_Fail" { + foundS1T2 = true + if res.Status != types.StatusFailure { + t.Errorf("S1_Test2_Fail status %s, want Failure", res.Status) + } + } + } + for _, res := range suite2Results { + if res.Spec.Name == "S2_Test1_Pass" { + foundS2T1 = true + if res.Status != types.StatusSuccess { + t.Errorf("S2_Test1_Pass status %s, want Success", res.Status) + } + } + if res.Spec.Name == "S2_Test2_SkipThis" { + foundS2T2 = true + if res.Status != types.StatusSkipped { + t.Errorf("S2_Test2_SkipThis status %s, want Skipped", res.Status) + } + } + } + if !foundS1T1 || !foundS1T2 || !foundS2T1 || !foundS2T2 { + t.Errorf("Not all tests were found in results map. S1T1:%v, S1T2:%v, S2T1:%v, S2T2:%v", foundS1T1, foundS1T2, foundS2T1, foundS2T2) + } +} diff --git a/internal/snapshot/service.go b/internal/snapshot/service.go new file mode 100644 index 0000000..932ece8 --- /dev/null +++ b/internal/snapshot/service.go @@ -0,0 +1,70 @@ +package snapshot + +import ( + "encoding/json" + "os" + "path" + "path/filepath" + "strings" + + apperrors "gitlab.com/technofab/nixtest/internal/errors" + "gitlab.com/technofab/nixtest/internal/util" +) + +// Service defines operations related to test snapshots +type Service interface { + GetPath(snapshotDir string, testName string) string + CreateFile(filePath string, data any) error + LoadFile(filePath string) (any, error) + Stat(name string) (os.FileInfo, error) +} + +type DefaultService struct{} + +func NewDefaultService() *DefaultService { + return &DefaultService{} +} + +// GetPath generates the canonical path for a snapshot file +func (s *DefaultService) GetPath(snapshotDir string, testName string) string { + fileName := filepath.ToSlash( + strings.ToLower(strings.ReplaceAll(testName, " ", "_")) + ".snap.json", + ) + return path.Join(snapshotDir, fileName) +} + +// CreateFile creates or updates a snapshot file with the given data +func (s *DefaultService) CreateFile(filePath string, data any) error { + jsonData, err := json.MarshalIndent(data, "", " ") + if err != nil { + return &apperrors.SnapshotCreateError{ + FilePath: filePath, + Err: &apperrors.JSONUnmarshalError{Source: "snapshot data for " + filePath, Err: err}, + } + } + + err = os.MkdirAll(path.Dir(filePath), 0777) + if err != nil { + return &apperrors.SnapshotCreateError{FilePath: filePath, Err: err} + } + + err = os.WriteFile(filePath, jsonData, 0644) + if err != nil { + return &apperrors.SnapshotCreateError{FilePath: filePath, Err: err} + } + return nil +} + +// LoadFile loads a snapshot file. +func (s *DefaultService) LoadFile(filePath string) (any, error) { + result, err := util.ParseFile[any](filePath) + if err != nil { + return nil, &apperrors.SnapshotLoadError{FilePath: filePath, Err: err} + } + return result, nil +} + +// Stat just wraps os.Stat +func (s *DefaultService) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} diff --git a/internal/snapshot/service_test.go b/internal/snapshot/service_test.go new file mode 100644 index 0000000..9a1f4bb --- /dev/null +++ b/internal/snapshot/service_test.go @@ -0,0 +1,151 @@ +package snapshot + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + apperrors "gitlab.com/technofab/nixtest/internal/errors" +) + +func TestDefaultService_GetPath(t *testing.T) { + service := NewDefaultService() + tests := []struct { + name string + snapshotDir string + testName string + want string + }{ + {"Simple name", "/tmp/snapshots", "TestSimple", "/tmp/snapshots/testsimple.snap.json"}, + {"Name with spaces", "snaps", "Test With Spaces", "snaps/test_with_spaces.snap.json"}, + {"Name with mixed case", "./data", "MyTestSERVICE", "data/mytestservice.snap.json"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + want := filepath.ToSlash(tt.want) // Normalize for comparison + got := service.GetPath(tt.snapshotDir, tt.testName) + if got != want { + t.Errorf("GetPath() = %v, want %v", got, want) + } + }) + } +} + +func TestDefaultService_CreateFileAndLoadFile(t *testing.T) { + service := NewDefaultService() + tempDir := t.TempDir() + filePath := service.GetPath(tempDir, "Test Snapshot Content") + + dataToWrite := map[string]any{ + "name": "test snapshot", + "value": float64(42), + "nested": map[string]any{"active": true}, + } + + t.Run("CreateFile", func(t *testing.T) { + err := service.CreateFile(filePath, dataToWrite) + if err != nil { + t.Fatalf("CreateFile() failed: %v", err) + } + + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read created snapshot file: %v", err) + } + + var readData map[string]any + if err := json.Unmarshal(content, &readData); err != nil { + t.Fatalf("Failed to unmarshal created snapshot content: %v", err) + } + if !reflect.DeepEqual(readData, dataToWrite) { + t.Errorf("CreateFile() content mismatch. Got %v, want %v", readData, dataToWrite) + } + }) + + t.Run("LoadFile", func(t *testing.T) { + // ensure file exists + if _, statErr := os.Stat(filePath); os.IsNotExist(statErr) { + if errCreate := service.CreateFile(filePath, dataToWrite); errCreate != nil { + t.Fatalf("Prerequisite CreateFile for LoadFile test failed: %v", errCreate) + } + } + + loadedData, err := service.LoadFile(filePath) + if err != nil { + t.Fatalf("LoadFile() failed: %v", err) + } + loadedMap, ok := loadedData.(map[string]any) + if !ok { + t.Fatalf("LoadFile() did not return a map, got %T", loadedData) + } + if !reflect.DeepEqual(loadedMap, dataToWrite) { + t.Errorf("LoadFile() content mismatch. Got %v, want %v", loadedMap, dataToWrite) + } + }) + + t.Run("LoadFile_NotExist", func(t *testing.T) { + nonExistentPath := service.GetPath(tempDir, "NonExistentSnapshot") + _, err := service.LoadFile(nonExistentPath) + if err == nil { + t.Error("LoadFile() expected error for non-existent file, got nil") + } else { + var loadErr *apperrors.SnapshotLoadError + if !errors.As(err, &loadErr) { + t.Errorf("LoadFile() wrong error type, got %T, want *apperrors.SnapshotLoadError", err) + } + var fileReadErr *apperrors.FileReadError + if !errors.As(loadErr.Err, &fileReadErr) { + t.Errorf("SnapshotLoadError wrong wrapped error type, got %T, want *apperrors.FileReadError", loadErr.Err) + } + if !strings.Contains(err.Error(), "failed to open") { + t.Errorf("LoadFile() error = %v, want error containing 'failed to open'", err) + } + } + }) + + t.Run("CreateFile_MarshalError", func(t *testing.T) { + err := service.CreateFile(filepath.Join(tempDir, "marshal_error.snap.json"), make(chan int)) + if err == nil { + t.Fatal("CreateFile expected error for unmarshalable data, got nil") + } + var createErr *apperrors.SnapshotCreateError + if !errors.As(err, &createErr) { + t.Fatalf("Wrong error type for marshal error, got %T", err) + } + var marshalErr *apperrors.JSONUnmarshalError + if !errors.As(createErr.Err, &marshalErr) { + t.Errorf("SnapshotCreateError did not wrap JSONUnmarshalError for marshal failure, got %T", createErr.Err) + } + }) +} + +func TestDefaultService_Stat(t *testing.T) { + service := NewDefaultService() + tempDir := t.TempDir() + + t.Run("File exists", func(t *testing.T) { + p := filepath.Join(tempDir, "exists.txt") + if err := os.WriteFile(p, []byte("content"), 0644); err != nil { + t.Fatal(err) + } + fi, err := service.Stat(p) + if err != nil { + t.Errorf("Stat() for existing file failed: %v", err) + } + if fi == nil || fi.Name() != "exists.txt" { + t.Errorf("Stat() returned incorrect FileInfo") + } + }) + + t.Run("File does not exist", func(t *testing.T) { + p := filepath.Join(tempDir, "notexists.txt") + _, err := service.Stat(p) + if !os.IsNotExist(err) { + t.Errorf("Stat() for non-existing file: got %v, want os.ErrNotExist", err) + } + }) +} diff --git a/internal/types/types.go b/internal/types/types.go new file mode 100644 index 0000000..6b39d7d --- /dev/null +++ b/internal/types/types.go @@ -0,0 +1,64 @@ +package types + +import "time" + +type TestType string + +const ( + TestTypeScript TestType = "script" + TestTypeUnit TestType = "unit" + TestTypeSnapshot TestType = "snapshot" +) + +type SuiteSpec struct { + Name string `json:"name"` + Tests []TestSpec `json:"tests"` +} + +type TestSpec struct { + Type TestType `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Expected any `json:"expected,omitempty"` + Actual any `json:"actual,omitempty"` + ActualDrv string `json:"actualDrv,omitempty"` + Script string `json:"script,omitempty"` + Pos string `json:"pos,omitempty"` + + Suite string +} + +type TestStatus int + +const ( + StatusSuccess TestStatus = iota + StatusFailure + StatusError + StatusSkipped +) + +func (ts TestStatus) String() string { + switch ts { + case StatusSuccess: + return "SUCCESS" + case StatusFailure: + return "FAILURE" + case StatusError: + return "ERROR" + case StatusSkipped: + return "SKIPPED" + default: + return "UNKNOWN" + } +} + +type TestResult struct { + Spec TestSpec + Status TestStatus + Duration time.Duration + ErrorMessage string + Expected string + Actual string +} + +type Results map[string][]TestResult diff --git a/internal/types/types_test.go b/internal/types/types_test.go new file mode 100644 index 0000000..37cc3b5 --- /dev/null +++ b/internal/types/types_test.go @@ -0,0 +1,24 @@ +package types + +import "testing" + +func TestTestStatus_String(t *testing.T) { + tests := []struct { + name string + status TestStatus + want string + }{ + {"Success", StatusSuccess, "SUCCESS"}, + {"Failure", StatusFailure, "FAILURE"}, + {"Error", StatusError, "ERROR"}, + {"Skipped", StatusSkipped, "SKIPPED"}, + {"Unknown", TestStatus(99), "UNKNOWN"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.status.String(); got != tt.want { + t.Errorf("TestStatus.String() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/util/util.go b/internal/util/util.go new file mode 100644 index 0000000..63e7252 --- /dev/null +++ b/internal/util/util.go @@ -0,0 +1,54 @@ +package util + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/akedrou/textdiff" + "github.com/akedrou/textdiff/myers" + apperrors "gitlab.com/technofab/nixtest/internal/errors" +) + +func ComputeDiff(expected, actual string) (string, error) { + // FIXME: ComputeEdits deprecated + edits := myers.ComputeEdits(expected, actual) + diff, err := textdiff.ToUnified("expected", "actual", expected, edits, 3) + if err != nil { + return "", err + } + // remove newline hint + diff = strings.ReplaceAll(diff, "\\ No newline at end of file\n", "") + return diff, nil +} + +// ParseFile reads and decodes a JSON file into the provided type +func ParseFile[T any](filePath string) (result T, err error) { + file, err := os.Open(filePath) + if err != nil { + return result, &apperrors.FileReadError{Path: filePath, Err: fmt.Errorf("failed to open: %w", err)} + } + defer file.Close() + + decoder := json.NewDecoder(file) + err = decoder.Decode(&result) + if err != nil { + return result, &apperrors.JSONUnmarshalError{Source: filePath, Err: fmt.Errorf("failed to decode: %w", err)} + } + return result, nil +} + +// PrefixLines adds a prefix to each line of the input string +func PrefixLines(input string, prefix string) string { + lines := strings.Split(input, "\n") + for i := range lines { + lines[i] = prefix + lines[i] + } + return strings.Join(lines, "\n") +} + +func IsString(value any) bool { + _, ok := value.(string) + return ok +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go new file mode 100644 index 0000000..0928c06 --- /dev/null +++ b/internal/util/util_test.go @@ -0,0 +1,234 @@ +package util + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + apperrors "gitlab.com/technofab/nixtest/internal/errors" +) + +func TestComputeDiff(t *testing.T) { + testCases := []struct { + name string + expected string + actual string + wantDiff string + wantErr bool + }{ + { + name: "identical strings", + expected: "line1\nline2\n", + actual: "line1\nline2\n", + wantDiff: "", + }, + { + name: "simple change", + expected: "hello\nworld\n", + actual: "hello\nuniverse\n", + wantDiff: `--- expected ++++ actual +@@ -1,2 +1,2 @@ + hello +-world ++universe +`, + }, + { + name: "addition", + expected: "line1\nline3\n", + actual: "line1\nline2\nline3\n", + wantDiff: `--- expected ++++ actual +@@ -1,2 +1,3 @@ + line1 ++line2 + line3 +`, + }, + { + name: "deletion", + expected: "line1\nline2\nline3\n", + actual: "line1\nline3\n", + wantDiff: `--- expected ++++ actual +@@ -1,3 +1,2 @@ + line1 +-line2 + line3 +`, + }, + { + name: "empty strings", + expected: "", + actual: "", + wantDiff: "", + }, + { + name: "expected empty, actual has content", + expected: "", + actual: "new content\n", + wantDiff: `--- expected ++++ actual +@@ -0,0 +1 @@ ++new content +`, + }, + { + name: "expected has content, actual empty", + expected: "old content\n", + actual: "", + wantDiff: `--- expected ++++ actual +@@ -1 +0,0 @@ +-old content +`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotDiff, err := ComputeDiff(tc.expected, tc.actual) + + if (err != nil) != tc.wantErr { + t.Errorf("ComputeDiff() error = %v, wantErr %v", err, tc.wantErr) + return + } + if err != nil { + return + } + + normalizedGotDiff := strings.ReplaceAll(gotDiff, "\r\n", "\n") + normalizedWantDiff := strings.ReplaceAll(tc.wantDiff, "\r\n", "\n") + + if normalizedGotDiff != normalizedWantDiff { + t.Errorf("ComputeDiff() mismatch:\n--- GOT DIFF ---\n%s\n--- WANT DIFF ---\n%s", normalizedGotDiff, normalizedWantDiff) + metaDiff, _ := ComputeDiff(normalizedWantDiff, normalizedGotDiff) + if metaDiff != "" { + t.Errorf("--- DIFF OF DIFFS ---\n%s", metaDiff) + } + } + }) + } +} + +func TestParseFile(t *testing.T) { + type sampleStruct struct { + Name string `json:"name"` + Value int `json:"value"` + } + + validJSON := `{"name": "test", "value": 123}` + invalidJSON := `{"name": "test", value: 123}` + + tempDir := t.TempDir() + + validFilePath := filepath.Join(tempDir, "valid.json") + if err := os.WriteFile(validFilePath, []byte(validJSON), 0644); err != nil { + t.Fatalf("Failed to write valid temp file: %v", err) + } + + invalidJSONFilePath := filepath.Join(tempDir, "invalid_content.json") + if err := os.WriteFile(invalidJSONFilePath, []byte(invalidJSON), 0644); err != nil { + t.Fatalf("Failed to write invalid temp file: %v", err) + } + + tests := []struct { + name string + filePath string + want sampleStruct + wantErr bool + wantErrType any // expected error type, e.g., (*apperrors.FileReadError)(nil) + errContains string + }{ + { + name: "Valid JSON file", + filePath: validFilePath, + want: sampleStruct{Name: "test", Value: 123}, + wantErr: false, + }, + { + name: "File not found", + filePath: filepath.Join(tempDir, "nonexistent.json"), + want: sampleStruct{}, + wantErr: true, + wantErrType: (*apperrors.FileReadError)(nil), + errContains: "failed to open", + }, + { + name: "Invalid JSON content", + filePath: invalidJSONFilePath, + want: sampleStruct{}, + wantErr: true, + wantErrType: (*apperrors.JSONUnmarshalError)(nil), + errContains: "failed to decode", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseFile[sampleStruct](tt.filePath) + if (err != nil) != tt.wantErr { + t.Errorf("ParseFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + if tt.wantErrType != nil && !errors.As(err, &tt.wantErrType) { + t.Errorf("ParseFile() error type = %T, want type %T", err, tt.wantErrType) + } + if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("ParseFile() error = %v, want error containing %v", err, tt.errContains) + } + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseFile() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPrefixLines(t *testing.T) { + tests := []struct { + name string + input string + prefix string + want string + }{ + {"Empty input", "", "| ", "| "}, + {"Single line", "hello", "| ", "| hello"}, + {"Multiple lines", "hello\nworld", "> ", "> hello\n> world"}, + {"Line with trailing newline", "hello\n", "- ", "- hello\n- "}, + {"Prefix with space", "line", "PREFIX ", "PREFIX line"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PrefixLines(tt.input, tt.prefix); got != tt.want { + t.Errorf("PrefixLines() = %q, want %q", got, tt.want) + } + }) + } +} +func TestIsString(t *testing.T) { + tests := []struct { + name string + value any + want bool + }{ + {"String value", "hello", true}, + {"Empty string", "", true}, + {"Integer value", 123, false}, + {"Boolean value", true, false}, + {"Nil value", nil, false}, + {"Struct value", struct{}{}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsString(tt.value); got != tt.want { + t.Errorf("IsString() = %v, want %v for value %v", got, tt.want, tt.value) + } + }) + } +} diff --git a/package.nix b/package.nix index 96d46c3..cce5967 100644 --- a/package.nix +++ b/package.nix @@ -12,11 +12,12 @@ buildGoModule { root = ./.; fileset = unions [ ./cmd + ./internal ./go.mod ./go.sum ]; }; subPackages = ["cmd/nixtest"]; - vendorHash = "sha256-Hmdtkp3UK/lveE2/U6FmKno38DxY+MMQlQuZFf1UBME="; + vendorHash = "sha256-6kARJgngmXielUoXukYdAA0QHk1mwLRvgKJhx+v1iSo="; meta.mainProgram = "nixtest"; }