From c9298b91f42a2f02842da6c41d34db342d4b3de6 Mon Sep 17 00:00:00 2001 From: technofab Date: Tue, 2 Sep 2025 13:06:52 +0200 Subject: [PATCH 1/2] chore!: default to pure mode, rename --pure flag to --impure for switching --- cmd/nixtest/main.go | 2 +- docs/cli.md | 2 +- docs/usage.md | 6 +++--- go.mod | 2 +- internal/config/config.go | 4 ++-- internal/config/config_test.go | 6 +++--- internal/nix/service.go | 10 +++++----- internal/nix/service_test.go | 6 +++--- internal/runner/runner.go | 4 ++-- internal/runner/runner_test.go | 8 ++++---- tests/lib_test.nix | 8 ++++---- 11 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cmd/nixtest/main.go b/cmd/nixtest/main.go index 5f141c1..fef12b1 100644 --- a/cmd/nixtest/main.go +++ b/cmd/nixtest/main.go @@ -60,7 +60,7 @@ func main() { SnapshotDir: appCfg.SnapshotDir, UpdateSnapshots: appCfg.UpdateSnapshots, SkipPattern: appCfg.SkipPattern, - PureEnv: appCfg.PureEnv, + ImpureEnv: appCfg.ImpureEnv, } testRunner, err := runner.New(runnerCfg, nixService, snapshotService) if err != nil { diff --git a/docs/cli.md b/docs/cli.md index 943939e..fd882a7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -4,7 +4,7 @@ Usage of nixtest: --junit string Path to generate JUNIT report to, leave empty to disable --no-color Disable coloring - --pure Unset all env vars before running script tests + --impure Don\'t unset all env vars before running script tests -s, --skip string Regular expression to skip tests (e.g., 'test-.*|.*-b') --snapshot-dir string Directory where snapshots are stored (default "./snapshots") -f, --tests string Path to JSON file containing tests (required) diff --git a/docs/usage.md b/docs/usage.md index a329017..ee7598c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -96,10 +96,10 @@ Examples: name = "script-test"; type = "script"; script = - # there are two modes, "default"/"impure" and "pure" + # there are two modes, "default"/"pure" and "impure" # in impure mode all env variables etc. from your current session are kept - # and are available to the test - # to make it more reproducible and cleaner, use --pure to switch to pure + # and are available to the test (using --impure). + # to make it more reproducible and cleaner, the default is pure # mode which will unset all env variables before running the test. That # requires you to set PATH yourself then: # diff --git a/go.mod b/go.mod index 4cb8b73..bb9e578 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitlab.com/technofab/nixtest -go 1.24.2 +go 1.23.0 require ( github.com/akedrou/textdiff v0.1.0 diff --git a/internal/config/config.go b/internal/config/config.go index 9eef9bb..61427fb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,7 +16,7 @@ type AppConfig struct { JunitPath string UpdateSnapshots bool SkipPattern string - PureEnv bool + ImpureEnv bool NoColor bool } @@ -29,7 +29,7 @@ func Load() AppConfig { flag.StringVar(&cfg.JunitPath, "junit", "", "Path to generate JUNIT report to, leave empty to disable") flag.BoolVarP(&cfg.UpdateSnapshots, "update-snapshots", "u", false, "Update all snapshots") flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')") - flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests") + flag.BoolVar(&cfg.ImpureEnv, "impure", false, "Don't unset all env vars before running script tests") flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring") helpRequested := flag.BoolP("help", "h", false, "Show this menu") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c864d90..249dd60 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -64,7 +64,7 @@ func TestLoad_CustomValues(t *testing.T) { "--junit", "report.xml", "-u", "--skip", "specific-test", - "--pure", + "--impure", "--no-color", } pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) // Reset flags @@ -83,7 +83,7 @@ func TestLoad_CustomValues(t *testing.T) { if cfg.SkipPattern != "specific-test" { t.Errorf("SkipPattern: got %s, want specific-test", cfg.SkipPattern) } - if !cfg.PureEnv { - t.Errorf("PureEnv: got %v, want true", cfg.PureEnv) + if !cfg.ImpureEnv { + t.Errorf("ImpureEnv: got %v, want true", cfg.ImpureEnv) } } diff --git a/internal/nix/service.go b/internal/nix/service.go index c0ec851..3c8effd 100644 --- a/internal/nix/service.go +++ b/internal/nix/service.go @@ -14,7 +14,7 @@ import ( type Service interface { BuildDerivation(derivation string) (string, error) BuildAndParseJSON(derivation string) (any, error) - BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) + BuildAndRunScript(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error) } type DefaultService struct { @@ -72,7 +72,7 @@ func (s *DefaultService) BuildAndParseJSON(derivation string) (any, error) { } // BuildAndRunScript builds a derivation and runs it as a script -func (s *DefaultService) BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) { +func (s *DefaultService) BuildAndRunScript(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error) { exitCode = -1 path, err := s.BuildDerivation(derivation) if err != nil { @@ -80,10 +80,10 @@ func (s *DefaultService) BuildAndRunScript(derivation string, pureEnv bool) (exi } var cmdArgs []string - if pureEnv { - cmdArgs = append([]string{"env", "-i"}, "bash", path) - } else { + if impureEnv { cmdArgs = []string{"bash", path} + } else { + cmdArgs = append([]string{"env", "-i"}, "bash", path) } cmd := s.commandExecutor(cmdArgs[0], cmdArgs[1:]...) diff --git a/internal/nix/service_test.go b/internal/nix/service_test.go index 3fee50a..0df0658 100644 --- a/internal/nix/service_test.go +++ b/internal/nix/service_test.go @@ -232,7 +232,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) { tests := []struct { name string derivation string - pureEnv bool + impureEnv bool mockBuildDrvOutput string mockBuildDrvError string mockBuildDrvExitCode string @@ -252,7 +252,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) { 0, "Hello", "ErrOut", false, nil, "", }, { - "Success pure", "script.drv#sh", true, mockScriptPath, "", "0", + "Success impure", "script.drv#sh", true, mockScriptPath, "", "0", "Hello", "ErrOut", "0", 0, "Hello", "ErrOut", false, nil, "", }, @@ -277,7 +277,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) { os.Setenv("MOCK_SCRIPT_STDERR", tt.mockScriptStderr) os.Setenv("MOCK_SCRIPT_EXIT_CODE", tt.mockScriptExitCode) - exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.pureEnv) + exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.impureEnv) if (err != nil) != tt.wantErr { t.Fatalf("BuildAndRunScript() error = %v, wantErr %v", err, tt.wantErr) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 6f250d1..99783e9 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -32,7 +32,7 @@ type Config struct { SnapshotDir string UpdateSnapshots bool SkipPattern string - PureEnv bool + ImpureEnv bool } func New(cfg Config, nixService nix.Service, snapService snapshot.Service) (*Runner, error) { @@ -181,7 +181,7 @@ func (r *Runner) handleUnitTest(result *types.TestResult, spec types.TestSpec, a // handleScriptTest processes script type tests func (r *Runner) handleScriptTest(result *types.TestResult, spec types.TestSpec) { - exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.PureEnv) + exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.ImpureEnv) if err != nil { result.Status = types.StatusError result.ErrorMessage = fmt.Sprintf("[system] failed to run script derivation %s: %v", spec.Script, err) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 1415dd8..2bbffa6 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -18,7 +18,7 @@ import ( type mockNixService struct { BuildDerivationFunc func(derivation string) (string, error) BuildAndParseJSONFunc func(derivation string) (any, error) - BuildAndRunScriptFunc func(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) + BuildAndRunScriptFunc func(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error) } func (m *mockNixService) BuildDerivation(d string) (string, error) { @@ -253,7 +253,7 @@ func TestRunner_executeTest(t *testing.T) { spec: types.TestSpec{Name: "ScriptSuccess", Type: types.TestTypeScript, Script: "script.sh"}, runnerConfig: Config{}, setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { - mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { + mNix.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) { return 0, "stdout", "stderr", nil } }, @@ -264,7 +264,7 @@ func TestRunner_executeTest(t *testing.T) { spec: types.TestSpec{Name: "ScriptFail", Type: types.TestTypeScript, Script: "script.sh"}, runnerConfig: Config{}, setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) { - mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { + mNix.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) { return 1, "out on fail", "err on fail", nil } }, @@ -313,7 +313,7 @@ func TestRunner_RunTests(t *testing.T) { mockSnapSvc := &mockSnapshotService{} mockNixSvc.BuildAndParseJSONFunc = func(derivation string) (any, error) { return "parsed", nil } - mockNixSvc.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { return 0, "", "", nil } + mockNixSvc.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) { return 0, "", "", nil } mockSnapSvc.StatFunc = func(name string) (os.FileInfo, error) { return mockFileInfo{}, nil } mockSnapSvc.LoadFileFunc = func(filePath string) (any, error) { return "snapshot", nil } mockSnapSvc.CreateFileFunc = func(filePath string, data any) error { return nil } diff --git a/tests/lib_test.nix b/tests/lib_test.nix index ae5b474..cdd8386 100644 --- a/tests/lib_test.nix +++ b/tests/lib_test.nix @@ -36,7 +36,7 @@ binary = (ntlib.mkBinary { nixtests = "stub"; - extraParams = "--pure"; + extraParams = "--impure"; }) + "/bin/nixtests:run"; in @@ -45,7 +45,7 @@ ${ntlib.helpers.path [pkgs.gnugrep]} ${ntlib.helpers.scriptHelpers} assert_file_contains ${binary} "nixtest" "should contain nixtest" - assert_file_contains ${binary} "--pure" "should contain --pure arg" + assert_file_contains ${binary} "--impure" "should contain --impure arg" assert_file_contains ${binary} "--tests=stub" "should contain --tests arg" run "${binary} --help" @@ -75,14 +75,14 @@ TMPDIR=$(tmpdir) # start without nix & env binaries to expect errors - run "${binary} --pure --junit=$TMPDIR/junit.xml" + run "${binary} --junit=$TMPDIR/junit.xml" assert "$exit_code -eq 2" "should exit 2" assert "-f $TMPDIR/junit.xml" "should create junit.xml" assert_contains "$output" "executable file not found" "nix should not be found in pure mode" # now add required deps ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]} - run "${binary} --pure --junit=$TMPDIR/junit2.xml" + run "${binary} --junit=$TMPDIR/junit2.xml" assert "$exit_code -eq 2" "should exit 2" assert "-f $TMPDIR/junit2.xml" "should create junit2.xml" assert_not_contains "$output" "executable file not found" "nix should now exist" From 5741109cc9ec2b6d41b56abd3f5bc51ed7a9a228 Mon Sep 17 00:00:00 2001 From: technofab Date: Tue, 2 Sep 2025 13:22:28 +0200 Subject: [PATCH 2/2] feat: run script tests in temp dirs for slightly better sandboxing --- internal/nix/service.go | 9 +++++++++ lib/scriptHelpers.sh | 10 ---------- tests/lib_test.nix | 12 ++++++------ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/internal/nix/service.go b/internal/nix/service.go index 3c8effd..40461f9 100644 --- a/internal/nix/service.go +++ b/internal/nix/service.go @@ -3,6 +3,7 @@ package nix import ( "bytes" "encoding/json" + "fmt" "os" "os/exec" "strings" @@ -79,6 +80,13 @@ func (s *DefaultService) BuildAndRunScript(derivation string, impureEnv bool) (e return exitCode, "", "", err } + // run scripts in a temporary directory + tempDir, err := os.MkdirTemp("", "nixtest-script-") + if err != nil { + return exitCode, "", "", &apperrors.ScriptExecutionError{Path: path, Err: fmt.Errorf("failed to create temporary directory: %w", err)} + } + defer os.RemoveAll(tempDir) + var cmdArgs []string if impureEnv { cmdArgs = []string{"bash", path} @@ -87,6 +95,7 @@ func (s *DefaultService) BuildAndRunScript(derivation string, impureEnv bool) (e } cmd := s.commandExecutor(cmdArgs[0], cmdArgs[1:]...) + cmd.Dir = tempDir var outBuf, errBuf bytes.Buffer cmd.Stdout = &outBuf cmd.Stderr = &errBuf diff --git a/lib/scriptHelpers.sh b/lib/scriptHelpers.sh index 0b59829..b1e3514 100644 --- a/lib/scriptHelpers.sh +++ b/lib/scriptHelpers.sh @@ -35,16 +35,6 @@ function assert_file_not_contains() { } } -function tmpdir() { - dir=$(mktemp -d) - trap "rm -rf $dir" EXIT - echo -n "$dir" -} -function tmpfile() { - file=$(mktemp) - trap "rm -f $file" EXIT - echo -n "$file" -} function run() { output=$($@ 2>&1) exit_code=$? diff --git a/tests/lib_test.nix b/tests/lib_test.nix index cdd8386..969e9a2 100644 --- a/tests/lib_test.nix +++ b/tests/lib_test.nix @@ -70,21 +70,21 @@ in # sh '' - ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]} + ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp pkgs.coreutils]} ${ntlib.helpers.scriptHelpers} + cp -r ${./../snapshots} snapshots - TMPDIR=$(tmpdir) # start without nix & env binaries to expect errors - run "${binary} --junit=$TMPDIR/junit.xml" + run "${binary} --junit=junit.xml" assert "$exit_code -eq 2" "should exit 2" - assert "-f $TMPDIR/junit.xml" "should create junit.xml" + assert "-f junit.xml" "should create junit.xml" assert_contains "$output" "executable file not found" "nix should not be found in pure mode" # now add required deps ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]} - run "${binary} --junit=$TMPDIR/junit2.xml" + run "${binary} --junit=junit2.xml" assert "$exit_code -eq 2" "should exit 2" - assert "-f $TMPDIR/junit2.xml" "should create junit2.xml" + assert "-f junit2.xml" "should create junit2.xml" assert_not_contains "$output" "executable file not found" "nix should now exist" assert_contains "$output" "suite-one" "should contain suite-one" assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"