feat: run script tests in temp dirs for slightly better sandboxing

cmd/nixtest/main.go

@@ -60,7 +60,7 @@ func main() {
 		SnapshotDir:     appCfg.SnapshotDir,
 		UpdateSnapshots: appCfg.UpdateSnapshots,
 		SkipPattern:     appCfg.SkipPattern,
-		PureEnv:         appCfg.PureEnv,
+		ImpureEnv:       appCfg.ImpureEnv,
 	}
 	testRunner, err := runner.New(runnerCfg, nixService, snapshotService)
 	if err != nil {

docs/cli.md

@@ -4,7 +4,7 @@
 Usage of nixtest:
       --junit string          Path to generate JUNIT report to, leave empty to disable
       --no-color              Disable coloring
-      --pure                  Unset all env vars before running script tests
+      --impure                Don\'t unset all env vars before running script tests
   -s, --skip string           Regular expression to skip tests (e.g., 'test-.*|.*-b')
       --snapshot-dir string   Directory where snapshots are stored (default "./snapshots")
   -f, --tests string          Path to JSON file containing tests (required)

docs/usage.md

@@ -96,10 +96,10 @@ Examples:
     name = "script-test";
     type = "script";
     script = 
-    # there are two modes, "default"/"impure" and "pure"
+    # there are two modes, "default"/"pure" and "impure"
     # in impure mode all env variables etc. from your current session are kept
-    # and are available to the test
-    # to make it more reproducible and cleaner, use --pure to switch to pure
+    # and are available to the test (using --impure).
+    # to make it more reproducible and cleaner, the default is pure
     # mode which will unset all env variables before running the test. That 
     # requires you to set PATH yourself then:
     # 

go.mod

@@ -1,6 +1,6 @@
 module gitlab.com/technofab/nixtest
 
-go 1.24.2
+go 1.23.0
 
 require (
 	github.com/akedrou/textdiff v0.1.0

internal/config/config.go

@@ -16,7 +16,7 @@ type AppConfig struct {
 	JunitPath       string
 	UpdateSnapshots bool
 	SkipPattern     string
-	PureEnv         bool
+	ImpureEnv       bool
 	NoColor         bool
 }
 
@@ -29,7 +29,7 @@ func Load() AppConfig {
 	flag.StringVar(&cfg.JunitPath, "junit", "", "Path to generate JUNIT report to, leave empty to disable")
 	flag.BoolVarP(&cfg.UpdateSnapshots, "update-snapshots", "u", false, "Update all snapshots")
 	flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')")
-	flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests")
+	flag.BoolVar(&cfg.ImpureEnv, "impure", false, "Don't unset all env vars before running script tests")
 	flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring")
 	helpRequested := flag.BoolP("help", "h", false, "Show this menu")
 

internal/config/config_test.go

@@ -64,7 +64,7 @@ func TestLoad_CustomValues(t *testing.T) {
 		"--junit", "report.xml",
 		"-u",
 		"--skip", "specific-test",
-		"--pure",
+		"--impure",
 		"--no-color",
 	}
 	pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) // Reset flags
@@ -83,7 +83,7 @@ func TestLoad_CustomValues(t *testing.T) {
 	if cfg.SkipPattern != "specific-test" {
 		t.Errorf("SkipPattern: got %s, want specific-test", cfg.SkipPattern)
 	}
-	if !cfg.PureEnv {
-		t.Errorf("PureEnv: got %v, want true", cfg.PureEnv)
+	if !cfg.ImpureEnv {
+		t.Errorf("ImpureEnv: got %v, want true", cfg.ImpureEnv)
 	}
 }

internal/nix/service.go

@@ -3,6 +3,7 @@ package nix
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"os"
 	"os/exec"
 	"strings"
@@ -14,7 +15,7 @@ import (
 type Service interface {
 	BuildDerivation(derivation string) (string, error)
 	BuildAndParseJSON(derivation string) (any, error)
-	BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error)
+	BuildAndRunScript(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error)
 }
 
 type DefaultService struct {
@@ -72,21 +73,29 @@ func (s *DefaultService) BuildAndParseJSON(derivation string) (any, error) {
 }
 
 // BuildAndRunScript builds a derivation and runs it as a script
-func (s *DefaultService) BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) {
+func (s *DefaultService) BuildAndRunScript(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error) {
 	exitCode = -1
 	path, err := s.BuildDerivation(derivation)
 	if err != nil {
 		return exitCode, "", "", err
 	}
 
+	// run scripts in a temporary directory
+	tempDir, err := os.MkdirTemp("", "nixtest-script-")
+	if err != nil {
+		return exitCode, "", "", &apperrors.ScriptExecutionError{Path: path, Err: fmt.Errorf("failed to create temporary directory: %w", err)}
+	}
+	defer os.RemoveAll(tempDir)
+
 	var cmdArgs []string
-	if pureEnv {
-		cmdArgs = append([]string{"env", "-i"}, "bash", path)
-	} else {
+	if impureEnv {
 		cmdArgs = []string{"bash", path}
+	} else {
+		cmdArgs = append([]string{"env", "-i"}, "bash", path)
 	}
 
 	cmd := s.commandExecutor(cmdArgs[0], cmdArgs[1:]...)
+	cmd.Dir = tempDir
 	var outBuf, errBuf bytes.Buffer
 	cmd.Stdout = &outBuf
 	cmd.Stderr = &errBuf

internal/nix/service_test.go

@@ -232,7 +232,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) {
 	tests := []struct {
 		name                 string
 		derivation           string
-		pureEnv              bool
+		impureEnv            bool
 		mockBuildDrvOutput   string
 		mockBuildDrvError    string
 		mockBuildDrvExitCode string
@@ -252,7 +252,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) {
 			0, "Hello", "ErrOut", false, nil, "",
 		},
 		{
-			"Success pure", "script.drv#sh", true, mockScriptPath, "", "0",
+			"Success impure", "script.drv#sh", true, mockScriptPath, "", "0",
 			"Hello", "ErrOut", "0",
 			0, "Hello", "ErrOut", false, nil, "",
 		},
@@ -277,7 +277,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) {
 			os.Setenv("MOCK_SCRIPT_STDERR", tt.mockScriptStderr)
 			os.Setenv("MOCK_SCRIPT_EXIT_CODE", tt.mockScriptExitCode)
 
-			exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.pureEnv)
+			exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.impureEnv)
 
 			if (err != nil) != tt.wantErr {
 				t.Fatalf("BuildAndRunScript() error = %v, wantErr %v", err, tt.wantErr)

internal/runner/runner.go

@@ -32,7 +32,7 @@ type Config struct {
 	SnapshotDir     string
 	UpdateSnapshots bool
 	SkipPattern     string
-	PureEnv         bool
+	ImpureEnv       bool
 }
 
 func New(cfg Config, nixService nix.Service, snapService snapshot.Service) (*Runner, error) {
@@ -181,7 +181,7 @@ func (r *Runner) handleUnitTest(result *types.TestResult, spec types.TestSpec, a
 
 // handleScriptTest processes script type tests
 func (r *Runner) handleScriptTest(result *types.TestResult, spec types.TestSpec) {
-	exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.PureEnv)
+	exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.ImpureEnv)
 	if err != nil {
 		result.Status = types.StatusError
 		result.ErrorMessage = fmt.Sprintf("[system] failed to run script derivation %s: %v", spec.Script, err)

internal/runner/runner_test.go

@@ -18,7 +18,7 @@ import (
 type mockNixService struct {
 	BuildDerivationFunc   func(derivation string) (string, error)
 	BuildAndParseJSONFunc func(derivation string) (any, error)
-	BuildAndRunScriptFunc func(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error)
+	BuildAndRunScriptFunc func(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error)
 }
 
 func (m *mockNixService) BuildDerivation(d string) (string, error) {
@@ -253,7 +253,7 @@ func TestRunner_executeTest(t *testing.T) {
 			spec:         types.TestSpec{Name: "ScriptSuccess", Type: types.TestTypeScript, Script: "script.sh"},
 			runnerConfig: Config{},
 			setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {
-				mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) {
+				mNix.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) {
 					return 0, "stdout", "stderr", nil
 				}
 			},
@@ -264,7 +264,7 @@ func TestRunner_executeTest(t *testing.T) {
 			spec:         types.TestSpec{Name: "ScriptFail", Type: types.TestTypeScript, Script: "script.sh"},
 			runnerConfig: Config{},
 			setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {
-				mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) {
+				mNix.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) {
 					return 1, "out on fail", "err on fail", nil
 				}
 			},
@@ -313,7 +313,7 @@ func TestRunner_RunTests(t *testing.T) {
 	mockSnapSvc := &mockSnapshotService{}
 
 	mockNixSvc.BuildAndParseJSONFunc = func(derivation string) (any, error) { return "parsed", nil }
-	mockNixSvc.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { return 0, "", "", nil }
+	mockNixSvc.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) { return 0, "", "", nil }
 	mockSnapSvc.StatFunc = func(name string) (os.FileInfo, error) { return mockFileInfo{}, nil }
 	mockSnapSvc.LoadFileFunc = func(filePath string) (any, error) { return "snapshot", nil }
 	mockSnapSvc.CreateFileFunc = func(filePath string, data any) error { return nil }

lib/scriptHelpers.sh

@@ -35,16 +35,6 @@ function assert_file_not_contains() {
   }
 }
 
-function tmpdir() {
-  dir=$(mktemp -d)
-  trap "rm -rf $dir" EXIT
-  echo -n "$dir"
-}
-function tmpfile() {
-  file=$(mktemp)
-  trap "rm -f $file" EXIT
-  echo -n "$file"
-}
 function run() {
   output=$($@ 2>&1)
   exit_code=$?

tests/lib_test.nix

@@ -36,7 +36,7 @@
           binary =
             (ntlib.mkBinary {
               nixtests = "stub";
-              extraParams = "--pure";
+              extraParams = "--impure";
             })
             + "/bin/nixtests:run";
         in
@@ -45,7 +45,7 @@
             ${ntlib.helpers.path [pkgs.gnugrep]}
             ${ntlib.helpers.scriptHelpers}
             assert_file_contains ${binary} "nixtest" "should contain nixtest"
-            assert_file_contains ${binary} "--pure" "should contain --pure arg"
+            assert_file_contains ${binary} "--impure" "should contain --impure arg"
             assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
 
             run "${binary} --help"
@@ -70,21 +70,21 @@
         in
           # sh
           ''
-            ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp]}
+            ${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp pkgs.coreutils]}
             ${ntlib.helpers.scriptHelpers}
+            cp -r ${./../snapshots} snapshots
 
-            TMPDIR=$(tmpdir)
             # start without nix & env binaries to expect errors
-            run "${binary} --pure --junit=$TMPDIR/junit.xml"
+            run "${binary} --junit=junit.xml"
             assert "$exit_code -eq 2" "should exit 2"
-            assert "-f $TMPDIR/junit.xml" "should create junit.xml"
+            assert "-f junit.xml" "should create junit.xml"
             assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
 
             # now add required deps
             ${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
-            run "${binary} --pure --junit=$TMPDIR/junit2.xml"
+            run "${binary} --junit=junit2.xml"
             assert "$exit_code -eq 2" "should exit 2"
-            assert "-f $TMPDIR/junit2.xml" "should create junit2.xml"
+            assert "-f junit2.xml" "should create junit2.xml"
             assert_not_contains "$output" "executable file not found" "nix should now exist"
             assert_contains "$output" "suite-one" "should contain suite-one"
             assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"