mirror of
https://gitlab.com/TECHNOFAB/nixtest.git
synced 2026-02-02 19:35:11 +01:00
chore: initial prototype
This commit is contained in:
commit
c1c19c324d
16 changed files with 1099 additions and 0 deletions
78
cmd/nixtest/display.go
Normal file
78
cmd/nixtest/display.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/jedib0t/go-pretty/v6/table"
|
||||
"github.com/jedib0t/go-pretty/v6/text"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func printErrors(results Results) {
|
||||
for _, suiteResults := range results {
|
||||
for _, result := range suiteResults {
|
||||
if result.Success {
|
||||
continue
|
||||
}
|
||||
fmt.Println(text.FgRed.Sprintf("⚠ Test \"%s\" failed:", result.Name))
|
||||
for line := range strings.Lines(result.Error) {
|
||||
fmt.Printf("%s %s", text.FgRed.Sprint("|"), line)
|
||||
}
|
||||
if result.Error == "" {
|
||||
fmt.Printf("- no output -")
|
||||
}
|
||||
fmt.Printf("\n\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printSummary(results Results, successCount int, totalCount int) {
|
||||
t := table.NewWriter()
|
||||
t.SetStyle(table.StyleLight)
|
||||
t.SetOutputMirror(os.Stdout)
|
||||
t.AppendHeader(table.Row{"Test", "Duration", "Pass", "File"})
|
||||
|
||||
log.Info().Msg("Summary:")
|
||||
|
||||
for suite, suiteResults := range results {
|
||||
suiteTotal := len(suiteResults)
|
||||
suiteSuccess := 0
|
||||
|
||||
for _, res := range suiteResults {
|
||||
if res.Success {
|
||||
suiteSuccess++
|
||||
}
|
||||
}
|
||||
|
||||
t.AppendRow(table.Row{
|
||||
text.Bold.Sprint(suite),
|
||||
"",
|
||||
fmt.Sprintf("%d/%d", suiteSuccess, suiteTotal),
|
||||
"",
|
||||
})
|
||||
for _, res := range suiteResults {
|
||||
symbol := "❌"
|
||||
if res.Success {
|
||||
symbol = "✅"
|
||||
}
|
||||
|
||||
t.AppendRow([]any{
|
||||
res.Name,
|
||||
fmt.Sprintf("%s", res.Duration),
|
||||
symbol,
|
||||
res.Pos,
|
||||
})
|
||||
}
|
||||
t.AppendSeparator()
|
||||
}
|
||||
|
||||
t.AppendFooter(table.Row{
|
||||
text.Bold.Sprint("TOTAL"),
|
||||
"",
|
||||
fmt.Sprintf("%d/%d", successCount, totalCount),
|
||||
"",
|
||||
})
|
||||
t.Render()
|
||||
}
|
||||
115
cmd/nixtest/junit.go
Normal file
115
cmd/nixtest/junit.go
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type JUnitReport struct {
|
||||
XMLName xml.Name `xml:"testsuite"`
|
||||
Name string `xml:"name,attr"`
|
||||
Tests int `xml:"tests,attr"`
|
||||
Failures int `xml:"failures,attr"`
|
||||
Time string `xml:"time,attr"` // in seconds
|
||||
Suites []JUnitTestSuite `xml:"testsuite"`
|
||||
}
|
||||
|
||||
type JUnitTestSuite struct {
|
||||
XMLName xml.Name `xml:"testsuite"`
|
||||
Name string `xml:"name,attr"`
|
||||
Tests int `xml:"tests,attr"`
|
||||
Failures int `xml:"failures,attr"`
|
||||
Time string `xml:"time,attr"` // in seconds
|
||||
TestCases []JUnitCase `xml:"testcase"`
|
||||
}
|
||||
|
||||
type JUnitCase struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Classname string `xml:"classname,attr"`
|
||||
Time string `xml:"time,attr"` // in seconds
|
||||
File string `xml:"file,attr,omitempty"`
|
||||
Line string `xml:"line,attr,omitempty"`
|
||||
Failure *string `xml:"failure,omitempty"`
|
||||
Error *string `xml:"error,omitempty"`
|
||||
}
|
||||
|
||||
func GenerateJUnitReport(name string, results Results) (string, error) {
|
||||
report := JUnitReport{
|
||||
Name: name,
|
||||
Tests: 0,
|
||||
Failures: 0,
|
||||
Suites: []JUnitTestSuite{},
|
||||
}
|
||||
|
||||
totalDuration := time.Duration(0)
|
||||
|
||||
for suiteName, suiteResults := range results {
|
||||
suite := JUnitTestSuite{
|
||||
Name: suiteName,
|
||||
Tests: len(suiteResults),
|
||||
Failures: 0,
|
||||
TestCases: []JUnitCase{},
|
||||
}
|
||||
|
||||
suiteDuration := time.Duration(0)
|
||||
|
||||
for _, result := range suiteResults {
|
||||
durationSeconds := fmt.Sprintf("%.3f", result.Duration.Seconds())
|
||||
totalDuration += result.Duration
|
||||
suiteDuration += result.Duration
|
||||
|
||||
testCase := JUnitCase{
|
||||
Name: result.Name,
|
||||
Classname: suiteName, // Use suite name as classname
|
||||
Time: durationSeconds,
|
||||
}
|
||||
|
||||
if result.Pos != "" {
|
||||
pos := strings.Split(result.Pos, ":")
|
||||
testCase.File = pos[0]
|
||||
testCase.Line = pos[1]
|
||||
}
|
||||
|
||||
if !result.Success {
|
||||
suite.Failures++
|
||||
report.Failures++
|
||||
testCase.Failure = &result.Error
|
||||
}
|
||||
|
||||
suite.TestCases = append(suite.TestCases, testCase)
|
||||
}
|
||||
suite.Time = fmt.Sprintf("%.3f", suiteDuration.Seconds())
|
||||
report.Suites = append(report.Suites, suite)
|
||||
report.Tests += len(suiteResults)
|
||||
}
|
||||
|
||||
report.Time = fmt.Sprintf("%.3f", totalDuration.Seconds())
|
||||
|
||||
output, err := xml.MarshalIndent(report, "", " ")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal XML: %w", err)
|
||||
}
|
||||
|
||||
return xml.Header + string(output), nil
|
||||
}
|
||||
|
||||
func GenerateJunitFile(path string, results Results) error {
|
||||
res, err := GenerateJUnitReport("nixtest", results)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate junit report: %w", err)
|
||||
}
|
||||
file, err := os.Create(*junitPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.WriteString(res)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write to file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
270
cmd/nixtest/main.go
Normal file
270
cmd/nixtest/main.go
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/sergi/go-diff/diffmatchpatch"
|
||||
flag "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
type SuiteSpec struct {
|
||||
Name string `json:"name"`
|
||||
Tests []TestSpec `json:"tests"`
|
||||
}
|
||||
|
||||
type TestSpec struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Expected any `json:"expected,omitempty"`
|
||||
Actual any `json:"actual,omitempty"`
|
||||
ActualDrv string `json:"actualDrv,omitempty"`
|
||||
Pos string `json:"pos,omitempty"`
|
||||
|
||||
Suite string
|
||||
}
|
||||
|
||||
type TestResult struct {
|
||||
Name string
|
||||
Success bool
|
||||
Error string
|
||||
Duration time.Duration
|
||||
Pos string
|
||||
Suite string
|
||||
}
|
||||
|
||||
type Results map[string][]TestResult
|
||||
|
||||
func buildAndParse(variable string) (any, error) {
|
||||
cmd := exec.Command(
|
||||
"nix",
|
||||
"build",
|
||||
variable+"^*",
|
||||
"--print-out-paths",
|
||||
"--no-link",
|
||||
)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
path := strings.TrimSpace(string(output))
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result any
|
||||
err = json.Unmarshal(data, &result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func runTest(spec TestSpec) TestResult {
|
||||
startTime := time.Now()
|
||||
result := TestResult{
|
||||
Name: spec.Name,
|
||||
Pos: spec.Pos,
|
||||
Suite: spec.Suite,
|
||||
Success: false,
|
||||
Error: "",
|
||||
}
|
||||
|
||||
var actual any
|
||||
var expected any
|
||||
|
||||
if spec.Type == "snapshot" {
|
||||
actual = spec.Actual
|
||||
filePath := path.Join(
|
||||
*snapshotDir,
|
||||
fmt.Sprintf("%s.snap.json", strings.ToLower(spec.Name)),
|
||||
)
|
||||
|
||||
if *updateSnapshots {
|
||||
createSnapshot(filePath, actual)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
|
||||
result.Error = "No Snapshot exists yet"
|
||||
goto end
|
||||
}
|
||||
|
||||
var err error
|
||||
expected, err = ParseFile[any](filePath)
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("[system] failed to parse snapshot: %v", err.Error())
|
||||
goto end
|
||||
}
|
||||
} else if spec.Type == "unit" {
|
||||
if spec.ActualDrv != "" {
|
||||
var err error
|
||||
actual, err = buildAndParse(spec.ActualDrv)
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("[system] failed to parse drv output: %v", err.Error())
|
||||
goto end
|
||||
}
|
||||
} else {
|
||||
actual = spec.Actual
|
||||
}
|
||||
expected = spec.Expected
|
||||
} else {
|
||||
log.Panic().Str("type", spec.Type).Msg("Invalid test type")
|
||||
}
|
||||
|
||||
if reflect.DeepEqual(actual, expected) {
|
||||
result.Success = true
|
||||
} else {
|
||||
dmp := diffmatchpatch.New()
|
||||
text1, err := json.MarshalIndent(actual, "", " ")
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("[system] failed to json marshal 'actual': %v", err.Error())
|
||||
goto end
|
||||
}
|
||||
text2, err := json.MarshalIndent(expected, "", " ")
|
||||
if err != nil {
|
||||
result.Error = fmt.Sprintf("[system] failed to json marshal 'expected': %v", err.Error())
|
||||
goto end
|
||||
}
|
||||
diffs := dmp.DiffMain(string(text1), string(text2), false)
|
||||
result.Error = fmt.Sprintf("Mismatch:\n%s", dmp.DiffPrettyText(diffs))
|
||||
}
|
||||
|
||||
end:
|
||||
result.Duration = time.Since(startTime)
|
||||
return result
|
||||
}
|
||||
|
||||
func createSnapshot(filePath string, actual any) error {
|
||||
jsonData, err := json.Marshal(actual)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.MkdirAll(path.Dir(filePath), 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.WriteFile(filePath, jsonData, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker to process TestSpec items
|
||||
func worker(jobs <-chan TestSpec, results chan<- TestResult, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for spec := range jobs {
|
||||
results <- runTest(spec)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
numWorkers *int = flag.Int("workers", 4, "Amount of tests to run in parallel")
|
||||
testsFile *string = flag.String("tests", "", "Path to JSON file containing tests")
|
||||
snapshotDir *string = flag.String(
|
||||
"snapshot-dir", "./snapshots", "Directory where snapshots are stored",
|
||||
)
|
||||
junitPath *string = flag.String(
|
||||
"junit", "", "Path to generate JUNIT report to, leave empty to disable",
|
||||
)
|
||||
updateSnapshots *bool = flag.Bool("update-snapshots", false, "Update all snapshots")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
||||
|
||||
log.Info().
|
||||
Int("workers", *numWorkers).
|
||||
Msg("Starting nixtest")
|
||||
|
||||
if _, err := os.Stat(*testsFile); errors.Is(err, os.ErrNotExist) {
|
||||
log.Error().Str("file", *testsFile).Msg("Tests file does not exist")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
parsedSpecs, err := ParseFile[[]SuiteSpec](*testsFile)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to load tests from file")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
totalTests := 0
|
||||
for _, suite := range parsedSpecs {
|
||||
totalTests += len(suite.Tests)
|
||||
}
|
||||
log.Info().
|
||||
Int("suites", len(parsedSpecs)).
|
||||
Int("tests", totalTests).
|
||||
Msg("Discovered suites")
|
||||
|
||||
jobsChan := make(chan TestSpec, totalTests)
|
||||
resultsChan := make(chan TestResult, totalTests)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 1; i <= *numWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go worker(jobsChan, resultsChan, &wg)
|
||||
}
|
||||
|
||||
for _, suite := range parsedSpecs {
|
||||
for _, test := range suite.Tests {
|
||||
test.Suite = suite.Name
|
||||
jobsChan <- test
|
||||
}
|
||||
}
|
||||
close(jobsChan)
|
||||
|
||||
wg.Wait()
|
||||
close(resultsChan)
|
||||
|
||||
results := map[string][]TestResult{}
|
||||
|
||||
successCount := 0
|
||||
|
||||
for r := range resultsChan {
|
||||
results[r.Suite] = append(results[r.Suite], r)
|
||||
if r.Success {
|
||||
successCount++
|
||||
}
|
||||
}
|
||||
|
||||
if *junitPath != "" {
|
||||
err = GenerateJunitFile(*junitPath, results)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to generate junit file")
|
||||
} else {
|
||||
log.Info().Str("path", *junitPath).Msg("Generated Junit report")
|
||||
}
|
||||
}
|
||||
|
||||
// print errors/logs of failed tests
|
||||
printErrors(results)
|
||||
|
||||
// show table summary
|
||||
printSummary(results, successCount, totalTests)
|
||||
|
||||
if successCount != totalTests {
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
24
cmd/nixtest/utils.go
Normal file
24
cmd/nixtest/utils.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func ParseFile[T any](filePath string) (result T, err error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to open file %s: %w", filePath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
decoder := json.NewDecoder(file)
|
||||
|
||||
err = decoder.Decode(&result)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to decode JSON from file %s: %w", filePath, err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue