Compare commits

..

No commits in common. "main" and "1.0.0" have entirely different histories.
main ... 1.0.0

31 changed files with 306 additions and 811 deletions

View file

@ -1,7 +0,0 @@
Copyright 2025 TECHNOFAB
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View file

@ -2,23 +2,19 @@
[![built with nix](https://img.shields.io/static/v1?logo=nixos&logoColor=white&label=&message=Built%20with%20Nix&color=41439a)](https://builtwithnix.org)
[![pipeline status](https://gitlab.com/TECHNOFAB/nixtest/badges/main/pipeline.svg)](https://gitlab.com/TECHNOFAB/nixtest/-/commits/main)
![License: MIT](https://img.shields.io/gitlab/license/technofab/nixtest)
![License: MIT](https://img.shields.io/gitlab/license/technofab/nix-gitlab-ci)
[![Latest Release](https://gitlab.com/TECHNOFAB/nixtest/-/badges/release.svg)](https://gitlab.com/TECHNOFAB/nixtest/-/releases)
[![Support me](https://img.shields.io/badge/Support-me-green)](https://tec.tf/#support)
[![Docs](https://img.shields.io/badge/Read-Docs-green)](https://nixtest.projects.tf)
Flexible test runner for testing Nix code, written in Go.
[![asciicast](https://asciinema.org/a/VS8uIiEQiPRtXbOreJqrxO7X6.svg)](https://asciinema.org/a/VS8uIiEQiPRtXbOreJqrxO7X6)
## Features
- Snapshot, Unit (equal checks) and Script-Tests (unit tests with assertions you could say)
- Supports testing against raw Nix code or derivation output
- Simple and easy to read summary of test results
![screenshot](./docs/images/summary_screenshot.png)
- Junit report support (eg. for displaying the results in GitLab etc.)
![screenshot](./docs/images/gitlab_junit_screenshot.png)
## Usage

View file

@ -60,7 +60,7 @@ func main() {
SnapshotDir: appCfg.SnapshotDir,
UpdateSnapshots: appCfg.UpdateSnapshots,
SkipPattern: appCfg.SkipPattern,
ImpureEnv: appCfg.ImpureEnv,
PureEnv: appCfg.PureEnv,
}
testRunner, err := runner.New(runnerCfg, nixService, snapshotService)
if err != nil {

View file

@ -3,11 +3,10 @@
```sh title="nix run .#nixtests:run -- --help"
Usage of nixtest:
--junit string Path to generate JUNIT report to, leave empty to disable
--no-color Disable coloring
--impure Don\'t unset all env vars before running script tests
-s, --skip string Regular expression to skip tests (e.g., 'test-.*|.*-b')
--pure Unset all env vars before running script tests
--skip string Regular expression to skip (e.g., 'test-.*|.*-b')
--snapshot-dir string Directory where snapshots are stored (default "./snapshots")
-f, --tests string Path to JSON file containing tests (required)
-u, --update-snapshots Update all snapshots
-w, --workers int Amount of tests to run in parallel (default 4)
--tests string Path to JSON file containing tests
--update-snapshots Update all snapshots
--workers int Amount of tests to run in parallel (default 4)
```

View file

@ -1,6 +1,4 @@
# Example Configs
- [nixtest itself](https://gitlab.com/TECHNOFAB/nixtest)
see `flake.nix` and `tests/`
- [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci)
see `tests/`
see tests/

BIN
docs/images/favicon.png Executable file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 173 KiB

BIN
docs/images/logo.png Executable file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="100" height="100" fill="none"><rect width="100" height="100" fill="#222" rx="12"/><rect width="100" height="100" fill="url(#a)" rx="12"/><path fill="#3FB950" d="M18 54.7a3 3 0 0 1 4.3 0l17 17c.5.4 1 .4 1.5 0L82.5 30a3 3 0 0 1 4.2 0l2 1.9a3 3 0 0 1 0 4.2L46.2 78.7l-4 4a3 3 0 0 1-4.2 0L16 60.9a3 3 0 0 1 0-4.3l2-1.9Z"/><path fill="url(#b)" d="m89 66.6-5.3 9.6H71l6.3 11-2.7 4.8h-5.5l-9-15.8H53l9.6-9.6H89Zm-76-4.8 1 1.2 6 5.9-3.5 6-5.5-9.4 2-3.7Zm3-5 .1-.2-.1.2Zm11.4-1.1-3-3.1a6 6 0 0 0-8.5 0l-1.8 1.8H4.7L2 49.6l2.7-4.8h18l6.5-11.2H40l-12.6 22ZM62.8 18l6.4-11h5.4l2.7 4.8-9 15.7 4.4 7.9-6.9 6.9L45.8 7 56.5 7l6.2 11.1Z"/><path fill="url(#c)" d="m54.9 91.9-10.7.1-3.1-5.5a6 6 0 0 0 3.2-1.7l4-4 .2-.2L55 92Zm-19.1-7.2L31.7 92h-5.3l-2.8-4.8L29 78l6.9 6.8ZM47 78l-.8.8-4 4-.1.1.1-.1 4-4 .8-.8ZM40.2 72Zm.5-.2Zm.3-4.5-1 .9-11.7-11.7 3.3-6L41 67.3Zm54.3-22.6 2.7 4.7-2.7 4.8H77.5l-6.3 11.2h-7.6l20.8-20.7h11Zm-54.8-22 12.7.1 5.4 9.5H12l5.3-9.5h12.5l-6.2-11L26.2 7h5.5l8.8 15.8Zm44.3 3.4a6 6 0 0 0-2.4.4l1.3-2.4 1.1 2Z"/><defs><linearGradient id="b" x1="76.7" x2="59.1" y1="47.2" y2="17.1" gradientUnits="userSpaceOnUse"><stop offset=".2" stop-color="#7EB1DD"/></linearGradient><linearGradient id="c" x1="63.4" x2="81.3" y1="70.3" y2="40.6" gradientUnits="userSpaceOnUse"><stop offset="1" stop-color="#5277C3"/></linearGradient><radialGradient id="a" cx="0" cy="0" r="1" gradientTransform="matrix(0 50 -50 0 50 50)" gradientUnits="userSpaceOnUse"><stop offset=".5" stop-color="#091A3D"/><stop offset="1" stop-color="#000819"/></radialGradient></defs></svg>

Before

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 90 KiB

View file

@ -7,6 +7,4 @@ Flexible test runner for testing Nix code, written in Go.
- Snapshot, Unit (equal checks) and Script-Tests (unit tests with assertions you could say)
- Supports testing against raw Nix code or derivation output
- Simple and easy to read summary of test results
![screenshot](./images/summary_screenshot.png)
- Junit report support (eg. for displaying the results in GitLab etc.)
![screenshot](./images/gitlab_junit_screenshot.png)

View file

@ -1,3 +0,0 @@
# Options
{% include 'options.md' %}

View file

@ -1,53 +0,0 @@
# Reference
## `flakeModule`
The `flakeModule` for [flake-parts](https://flake.parts).
## `lib`
### `module`
The nix module for validation of inputs etc.
Used internally by `mkNixtestConfig`.
### `autodiscover`
```nix
autodiscover {
dir,
pattern ? ".*_test.nix",
}
```
Finds all test files in `dir` matching `pattern`.
Returns a list of modules (can be passed to `mkNixtest`'s `modules` arg).
### `mkNixtestConfig`
```nix
mkNixtestConfig {
modules,
args ? {},
}
```
Evaluates the test `modules`.
`args` are passed to the modules using `_module.args = args`.
**Noteworthy attributes**:
- `app`: nixtest wrapper
- `finalConfigJson`: derivation containing the tests json file
### `mkNixtest`
```nix
mkNixtest {
modules,
args ? {},
}
```
Creates the nixtest wrapper, using the tests in `modules`.
Basically `(mkNixtestConfig <arguments>).app`.

View file

@ -1,15 +0,0 @@
.md-header__button.md-logo {
margin: 0;
padding-top: .2rem;
padding-bottom: .2rem;
}
[dir="ltr"] .md-header__title {
margin-left: 0;
}
.md-header__button.md-logo img,
.md-header__button.md-logo svg {
height: 2rem;
}

View file

@ -2,9 +2,6 @@
## Flake Module
The easiest way to use Nixtest is probably using the flakeModule.
Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`:
```nix
{
inputs.nixtest.url = "gitlab:TECHNOFAB/nixtest?dir=lib";
@ -37,25 +34,10 @@ Just import `nixtest.flakeModule`, then define suites and tests in `perSystem`:
## Library
You can also use the lib directly, like this for example:
You can also integrate nixtest in your own workflow by using the lib functions directly.
Check out `flakeModule.nix` to see how it's used there.
```nix
packages.tests = ntlib.mkNixtest {
modules = ntlib.autodiscover {dir = ./tests;};
args = {
inherit pkgs ntlib;
};
};
```
This will auto-discover all test files ending with `_test.nix`.
See [reference](reference.md) for all params to `autodiscover`.
`ntlib` can be defined like this:
```nix
ntlib = inputs.nixtests.lib {inherit pkgs;};
```
<!-- TODO: more detailed? -->
## Define Tests
@ -96,30 +78,22 @@ Examples:
name = "script-test";
type = "script";
script =
# there are two modes, "default"/"pure" and "impure"
# there are two modes, "default"/"impure" and "pure"
# in impure mode all env variables etc. from your current session are kept
# and are available to the test (using --impure).
# to make it more reproducible and cleaner, the default is pure
# and are available to the test
# to make it more reproducible and cleaner, use --pure to switch to pure
# mode which will unset all env variables before running the test. That
# requires you to set PATH yourself then:
#
# ''
# export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
# grep -q "test" ${builtins.toFile "test" "test"}
# '';
#
# you can also use the helpers to make it nicer to read:
''
${ntlib.helpers.path [pkgs.gnugrep]}
${ntlib.helpers.scriptHelpers} # this adds helpers like assert etc.
assert_file_contains ${builtins.toFile "test" "test"} "test" "file should contain 'test'"
export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
grep -q "test" ${builtins.toFile "test" "test"}
'';
}
{
name = "pretty-test";
# by default it uses json to serialize and compare the values. Derivations
# and functions don't really work that way though, so you can also use
# "pretty" to use lib.generators.toPretty
# "pretty" to use lib.generators.pretty
format = "pretty";
# you can also set the pos here
pos = __curPos;
@ -128,7 +102,3 @@ Examples:
}
]
```
!!! note
for more examples see [examples](./examples.md)

22
flake.lock generated
View file

@ -169,6 +169,21 @@
"type": "github"
}
},
"mkdocs-material-umami": {
"locked": {
"lastModified": 1745840856,
"narHash": "sha256-1Ad1JTMQMP6YsoIKAA+SBCE15qWrYkGue9/lXOLnu9I=",
"owner": "technofab",
"repo": "mkdocs-material-umami",
"rev": "3ac9b194450f6b779c37b8d16fec640198e5cd0a",
"type": "gitlab"
},
"original": {
"owner": "technofab",
"repo": "mkdocs-material-umami",
"type": "gitlab"
}
},
"nix": {
"inputs": {
"flake-compat": [
@ -240,11 +255,11 @@
"nix-mkdocs": {
"locked": {
"dir": "lib",
"lastModified": 1757055638,
"narHash": "sha256-KHYSkEreFe4meXzSdEbknC/HwaQSNClQkc8vzHlAsMM=",
"lastModified": 1745841841,
"narHash": "sha256-297zPQbUlc7ZAYDoaD6mCmQxCC3Tr4YOKekRF1ArZ7g=",
"owner": "technofab",
"repo": "nixmkdocs",
"rev": "7840a5febdbeaf2da90babf6c94b3d0929d2bf74",
"rev": "c7e3c3b13ded25818e9789938387bba6f2cde690",
"type": "gitlab"
},
"original": {
@ -353,6 +368,7 @@
"inputs": {
"devenv": "devenv",
"flake-parts": "flake-parts_2",
"mkdocs-material-umami": "mkdocs-material-umami",
"nix-devtools": "nix-devtools",
"nix-gitlab-ci": "nix-gitlab-ci",
"nix-mkdocs": "nix-mkdocs",

220
flake.nix
View file

@ -11,13 +11,13 @@
inputs.nix-gitlab-ci.flakeModule
inputs.nix-devtools.flakeModule
inputs.nix-mkdocs.flakeModule
./lib/flakeModule.nix
];
systems = import systems;
flake = {};
perSystem = {
lib,
pkgs,
self',
config,
...
}: {
@ -37,12 +37,9 @@
};
devenv.shells.default = {
containers = pkgs.lib.mkForce {};
packages = with pkgs; [gore go-junit-report];
packages = with pkgs; [gopls gore go-junit-report];
languages.go = {
enable = true;
enableHardeningWorkaround = true;
};
languages.go.enable = true;
pre-commit.hooks = {
treefmt = {
@ -66,58 +63,183 @@
};
};
docs."default".config = {
nixtest = {
skip = "skip.*d";
suites = {
"suite-one" = {
pos = __curPos;
tests = [
{
name = "test-one";
# required to figure out file and line, but optional
expected = 1;
actual = 1;
}
{
name = "fail";
expected = 0;
actual = "meow";
}
{
name = "snapshot-test";
type = "snapshot";
actual = "test";
}
{
name = "test-snapshot-drv";
type = "snapshot";
actualDrv = pkgs.runCommand "test-snapshot" {} ''
echo '"snapshot drv"' > $out
'';
}
{
name = "test-error-drv";
expected = null;
actualDrv = pkgs.runCommand "test-error-drv" {} ''
echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
exit 1
'';
}
{
name = "test-script";
type = "script";
script = ''
echo Test something here
# required in pure mode:
export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
grep -q "test" ${builtins.toFile "test" "test"}
'';
}
];
};
"other-suite".tests = [
{
name = "obj-snapshot";
type = "snapshot";
pos = __curPos;
actual = {hello = "world";};
}
{
name = "pretty-snapshot";
type = "snapshot";
format = "pretty";
pos = __curPos;
actual = {
example = args: {};
example2 = {
drv = pkgs.hello;
};
};
}
{
name = "pretty-unit";
format = "pretty";
pos = __curPos;
expected = pkgs.hello;
actual = pkgs.hello;
}
{
name = "test-drv";
pos = __curPos;
expected = {a = "b";};
actualDrv = pkgs.runCommand "test-something" {} ''
echo "Simulating taking some time"
sleep 1
echo '{"a":"b"}' > $out
'';
}
{
name = "skipped";
expected = null;
actual = null;
}
];
};
};
doc = {
path = ./docs;
material = {
enable = true;
colors = {
primary = "green";
accent = "light green";
};
umami = {
enable = true;
src = "https://analytics.tf/umami";
siteId = "716d1869-9342-4b62-a770-e15d2d5c807d";
domains = ["nixtest.projects.tf"];
};
};
macros = {
enable = true;
includeDir = toString self'.packages.optionsDocs;
};
deps = pp: [
pp.mkdocs-material
(pp.callPackage inputs.mkdocs-material-umami {})
];
config = {
site_name = "Nixtest";
site_url = "https://nixtest.projects.tf";
repo_name = "TECHNOFAB/nixtest";
repo_url = "https://gitlab.com/TECHNOFAB/nixtest";
extra_css = ["style.css"];
edit_uri = "edit/main/docs/";
theme = {
name = "material";
features = ["content.code.copy" "content.action.edit"];
icon.repo = "simple/gitlab";
logo = "images/logo.svg";
favicon = "images/logo.svg";
logo = "images/logo.png";
favicon = "images/favicon.png";
palette = [
{
scheme = "default";
media = "(prefers-color-scheme: light)";
primary = "green";
accent = "light green";
toggle = {
icon = "material/brightness-7";
name = "Switch to dark mode";
};
}
{
scheme = "slate";
media = "(prefers-color-scheme: dark)";
primary = "green";
accent = "light green";
toggle = {
icon = "material/brightness-4";
name = "Switch to light mode";
};
}
];
};
plugins = ["search" "material-umami"];
nav = [
{"Introduction" = "index.md";}
{"Usage" = "usage.md";}
{"Reference" = "reference.md";}
{"CLI" = "cli.md";}
{"Example Configs" = "examples.md";}
{"Options" = "options.md";}
];
markdown_extensions = [
"pymdownx.superfences"
"admonition"
];
extra.analytics = {
provider = "umami";
site_id = "716d1869-9342-4b62-a770-e15d2d5c807d";
src = "https://analytics.tf/umami";
domains = "nixtest.projects.tf";
feedback = {
title = "Was this page helpful?";
ratings = [
{
icon = "material/thumb-up-outline";
name = "This page is helpful";
data = "good";
note = "Thanks for your feedback!";
}
{
icon = "material/thumb-down-outline";
name = "This page could be improved";
data = "bad";
note = "Thanks for your feedback! Please leave feedback by creating an issue :)";
}
];
};
};
};
};
ci = {
stages = ["test" "build" "deploy"];
jobs = {
"test:lib" = {
"test" = {
stage = "test";
script = [
"nix run .#tests -- --junit=junit.xml"
"nix run .#nixtests:run -- --junit=junit.xml"
];
allow_failure = true;
artifacts = {
@ -178,36 +300,7 @@
};
};
packages = let
ntlib = import ./lib {inherit pkgs lib;};
doclib = inputs.nix-mkdocs.lib {inherit lib pkgs;};
in rec {
default = pkgs.callPackage ./package.nix {};
tests = ntlib.mkNixtest {
modules = ntlib.autodiscover {dir = ./tests;};
args = {
inherit pkgs ntlib;
};
};
optionsDoc = doclib.mkOptionDocs {
module = {
_module.args.pkgs = pkgs;
imports = [
ntlib.module
];
};
roots = [
{
url = "https://gitlab.com/TECHNOFAB/nixtest/-/blob/main/lib";
path = toString ./lib;
}
];
};
optionsDocs = pkgs.runCommand "options-docs" {} ''
mkdir -p $out
ln -s ${optionsDoc} $out/options.md
'';
};
packages.default = pkgs.callPackage ./package.nix {};
};
};
@ -222,6 +315,7 @@
nix-gitlab-ci.url = "gitlab:technofab/nix-gitlab-ci/2.0.1?dir=lib";
nix-devtools.url = "gitlab:technofab/nix-devtools?dir=lib";
nix-mkdocs.url = "gitlab:technofab/nixmkdocs?dir=lib";
mkdocs-material-umami.url = "gitlab:technofab/mkdocs-material-umami";
};
nixConfig = {

2
go.mod
View file

@ -1,6 +1,6 @@
module gitlab.com/technofab/nixtest
go 1.23.0
go 1.24.2
require (
github.com/akedrou/textdiff v0.1.0

View file

@ -1,9 +1,6 @@
package config
import (
"fmt"
"os"
"github.com/jedib0t/go-pretty/v6/text"
"github.com/rs/zerolog/log"
flag "github.com/spf13/pflag"
@ -16,7 +13,7 @@ type AppConfig struct {
JunitPath string
UpdateSnapshots bool
SkipPattern string
ImpureEnv bool
PureEnv bool
NoColor bool
}
@ -29,18 +26,11 @@ func Load() AppConfig {
flag.StringVar(&cfg.JunitPath, "junit", "", "Path to generate JUNIT report to, leave empty to disable")
flag.BoolVarP(&cfg.UpdateSnapshots, "update-snapshots", "u", false, "Update all snapshots")
flag.StringVarP(&cfg.SkipPattern, "skip", "s", "", "Regular expression to skip tests (e.g., 'test-.*|.*-b')")
flag.BoolVar(&cfg.ImpureEnv, "impure", false, "Don't unset all env vars before running script tests")
flag.BoolVar(&cfg.PureEnv, "pure", false, "Unset all env vars before running script tests")
flag.BoolVar(&cfg.NoColor, "no-color", false, "Disable coloring")
helpRequested := flag.BoolP("help", "h", false, "Show this menu")
flag.Parse()
if *helpRequested {
fmt.Println("Usage of nixtest:")
flag.PrintDefaults()
os.Exit(0)
}
if cfg.TestsFile == "" {
log.Panic().Msg("Tests file path (-f or --tests) is required.")
}

View file

@ -64,7 +64,7 @@ func TestLoad_CustomValues(t *testing.T) {
"--junit", "report.xml",
"-u",
"--skip", "specific-test",
"--impure",
"--pure",
"--no-color",
}
pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) // Reset flags
@ -83,7 +83,7 @@ func TestLoad_CustomValues(t *testing.T) {
if cfg.SkipPattern != "specific-test" {
t.Errorf("SkipPattern: got %s, want specific-test", cfg.SkipPattern)
}
if !cfg.ImpureEnv {
t.Errorf("ImpureEnv: got %v, want true", cfg.ImpureEnv)
if !cfg.PureEnv {
t.Errorf("PureEnv: got %v, want true", cfg.PureEnv)
}
}

View file

@ -3,7 +3,6 @@ package nix
import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
@ -15,7 +14,7 @@ import (
type Service interface {
BuildDerivation(derivation string) (string, error)
BuildAndParseJSON(derivation string) (any, error)
BuildAndRunScript(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error)
BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error)
}
type DefaultService struct {
@ -73,29 +72,21 @@ func (s *DefaultService) BuildAndParseJSON(derivation string) (any, error) {
}
// BuildAndRunScript builds a derivation and runs it as a script
func (s *DefaultService) BuildAndRunScript(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error) {
func (s *DefaultService) BuildAndRunScript(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error) {
exitCode = -1
path, err := s.BuildDerivation(derivation)
if err != nil {
return exitCode, "", "", err
}
// run scripts in a temporary directory
tempDir, err := os.MkdirTemp("", "nixtest-script-")
if err != nil {
return exitCode, "", "", &apperrors.ScriptExecutionError{Path: path, Err: fmt.Errorf("failed to create temporary directory: %w", err)}
}
defer os.RemoveAll(tempDir)
var cmdArgs []string
if impureEnv {
cmdArgs = []string{"bash", path}
} else {
if pureEnv {
cmdArgs = append([]string{"env", "-i"}, "bash", path)
} else {
cmdArgs = []string{"bash", path}
}
cmd := s.commandExecutor(cmdArgs[0], cmdArgs[1:]...)
cmd.Dir = tempDir
var outBuf, errBuf bytes.Buffer
cmd.Stdout = &outBuf
cmd.Stderr = &errBuf

View file

@ -232,7 +232,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) {
tests := []struct {
name string
derivation string
impureEnv bool
pureEnv bool
mockBuildDrvOutput string
mockBuildDrvError string
mockBuildDrvExitCode string
@ -252,7 +252,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) {
0, "Hello", "ErrOut", false, nil, "",
},
{
"Success impure", "script.drv#sh", true, mockScriptPath, "", "0",
"Success pure", "script.drv#sh", true, mockScriptPath, "", "0",
"Hello", "ErrOut", "0",
0, "Hello", "ErrOut", false, nil, "",
},
@ -277,7 +277,7 @@ func TestDefaultService_BuildAndRunScript(t *testing.T) {
os.Setenv("MOCK_SCRIPT_STDERR", tt.mockScriptStderr)
os.Setenv("MOCK_SCRIPT_EXIT_CODE", tt.mockScriptExitCode)
exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.impureEnv)
exitCode, stdout, stderr, err := service.BuildAndRunScript(tt.derivation, tt.pureEnv)
if (err != nil) != tt.wantErr {
t.Fatalf("BuildAndRunScript() error = %v, wantErr %v", err, tt.wantErr)

View file

@ -32,7 +32,7 @@ type Config struct {
SnapshotDir string
UpdateSnapshots bool
SkipPattern string
ImpureEnv bool
PureEnv bool
}
func New(cfg Config, nixService nix.Service, snapService snapshot.Service) (*Runner, error) {
@ -181,7 +181,7 @@ func (r *Runner) handleUnitTest(result *types.TestResult, spec types.TestSpec, a
// handleScriptTest processes script type tests
func (r *Runner) handleScriptTest(result *types.TestResult, spec types.TestSpec) {
exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.ImpureEnv)
exitCode, stdout, stderrStr, err := r.nixService.BuildAndRunScript(spec.Script, r.config.PureEnv)
if err != nil {
result.Status = types.StatusError
result.ErrorMessage = fmt.Sprintf("[system] failed to run script derivation %s: %v", spec.Script, err)

View file

@ -18,7 +18,7 @@ import (
type mockNixService struct {
BuildDerivationFunc func(derivation string) (string, error)
BuildAndParseJSONFunc func(derivation string) (any, error)
BuildAndRunScriptFunc func(derivation string, impureEnv bool) (exitCode int, stdout string, stderr string, err error)
BuildAndRunScriptFunc func(derivation string, pureEnv bool) (exitCode int, stdout string, stderr string, err error)
}
func (m *mockNixService) BuildDerivation(d string) (string, error) {
@ -253,7 +253,7 @@ func TestRunner_executeTest(t *testing.T) {
spec: types.TestSpec{Name: "ScriptSuccess", Type: types.TestTypeScript, Script: "script.sh"},
runnerConfig: Config{},
setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {
mNix.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) {
mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) {
return 0, "stdout", "stderr", nil
}
},
@ -264,7 +264,7 @@ func TestRunner_executeTest(t *testing.T) {
spec: types.TestSpec{Name: "ScriptFail", Type: types.TestTypeScript, Script: "script.sh"},
runnerConfig: Config{},
setupMockServices: func(t *testing.T, mNix *mockNixService, mSnap *mockSnapshotService, s types.TestSpec, c Config) {
mNix.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) {
mNix.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) {
return 1, "out on fail", "err on fail", nil
}
},
@ -313,7 +313,7 @@ func TestRunner_RunTests(t *testing.T) {
mockSnapSvc := &mockSnapshotService{}
mockNixSvc.BuildAndParseJSONFunc = func(derivation string) (any, error) { return "parsed", nil }
mockNixSvc.BuildAndRunScriptFunc = func(derivation string, impureEnv bool) (int, string, string, error) { return 0, "", "", nil }
mockNixSvc.BuildAndRunScriptFunc = func(derivation string, pureEnv bool) (int, string, string, error) { return 0, "", "", nil }
mockSnapSvc.StatFunc = func(name string) (os.FileInfo, error) { return mockFileInfo{}, nil }
mockSnapSvc.LoadFileFunc = func(filePath string) (any, error) { return "snapshot", nil }
mockSnapSvc.CreateFileFunc = func(filePath string, data any) error { return nil }

View file

@ -1,70 +1,62 @@
{
pkgs,
lib ? pkgs.lib,
self ? "",
...
}: let
inherit (lib) evalModules toList;
in rec {
helpers = import ./testHelpers.nix {inherit lib;};
mkBinary = {
nixtests,
extraParams,
}: {
mkTest = {
type ? "unit",
name,
description ? "",
format ? "json",
expected ? null,
actual ? null,
actualDrv ? null,
script ? null,
pos ? null,
}: let
program = pkgs.callPackage ../package.nix {};
fileRelative = lib.removePrefix ((toString self) + "/") pos.file;
actual' =
if format == "json"
then actual
else lib.generators.toPretty {} actual;
expected' =
if format == "json"
then expected
else lib.generators.toPretty {} expected;
in
(pkgs.writeShellScriptBin "nixtests:run" ''
${program}/bin/nixtest --tests=${nixtests} ${extraParams} "$@"
'')
// {
tests = nixtests;
assert lib.assertMsg (!(type == "script" && script == null)) "test ${name} has type 'script' but no script was passed"; {
inherit type name description;
actual = actual';
expected = expected';
# discard string context, otherwise it's being built instantly which we don't want
actualDrv = builtins.unsafeDiscardStringContext (actualDrv.drvPath or "");
script =
if script != null
then
builtins.unsafeDiscardStringContext
(pkgs.writeShellScript "nixtest-${name}" ''
# show which line failed the test
set -x
${script}
'').drvPath
else null;
pos =
if pos == null
then ""
else "${fileRelative}:${toString pos.line}";
};
mkSuite = name: tests: {
inherit name tests;
};
exportSuites = suites: let
suitesList =
if builtins.isList suites
then suites
else [suites];
suitesMapped = builtins.toJSON suitesList;
testsMapped = builtins.toJSON suitesList;
in
pkgs.runCommand "tests.json" {} ''
echo '${suitesMapped}' > $out
echo '${testsMapped}' > $out
'';
module = import ./module.nix {inherit lib pkgs;};
autodiscover = {
dir,
pattern ? ".*_test.nix",
}: let
files = builtins.readDir dir;
matchingFiles = builtins.filter (name: builtins.match pattern name != null) (builtins.attrNames files);
imports = map (file:
if builtins.isString dir
then (builtins.unsafeDiscardStringContext dir) + "/${file}"
else /${dir}/${file})
matchingFiles;
in {
inherit imports;
# automatically set the base so test filepaths are easier to read
config.base = builtins.toString dir + "/";
};
mkNixtestConfig = {
modules,
args ? {},
...
}:
(evalModules {
modules =
(toList modules)
++ [
module
{
_module.args = args;
}
];
}).config;
mkNixtest = args: (mkNixtestConfig args).app;
}

View file

@ -15,14 +15,56 @@ in {
nixtests-lib = import ./. {inherit pkgs self;};
in {
options.nixtest = mkOption {
type = types.submodule (nixtests-lib.module);
type = types.submodule ({...}: {
options = {
skip = mkOption {
type = types.str;
default = "";
description = "Which tests to skip (regex)";
};
suites = mkOption {
type = types.attrsOf (types.submodule {
options = {
tests = mkOption {
type = types.listOf types.attrs;
default = [];
};
pos = mkOption {
type = types.nullOr types.attrs;
default = null;
};
};
});
default = {};
};
};
});
default = {};
};
config.nixtest.base = toString self + "/";
config.legacyPackages = {
"nixtests" = config.nixtest.finalConfigJson;
"nixtests:run" = config.nixtest.app;
config.legacyPackages = rec {
"nixtests" = let
suites = map (suiteName: let
suite = builtins.getAttr suiteName config.nixtest.suites;
in
nixtests-lib.mkSuite
suiteName
(map (test:
nixtests-lib.mkTest ({
# default pos to suite's pos if given
pos = suite.pos;
}
// test))
suite.tests))
(builtins.attrNames config.nixtest.suites);
in
nixtests-lib.exportSuites suites;
"nixtests:run" = let
program = pkgs.callPackage ./../package.nix {};
in
pkgs.writeShellScriptBin "nixtests:run" ''
${program}/bin/nixtest --tests=${nixtests} --skip="${config.nixtest.skip}" "$@"
'';
};
}
);

View file

@ -1,268 +0,0 @@
{
pkgs,
lib,
...
}: let
inherit
(lib)
mkOptionType
mkOption
types
filterAttrs
isType
removePrefix
assertMsg
generators
literalExpression
;
nixtest-lib = import ./default.nix {inherit pkgs lib;};
unsetType = mkOptionType {
name = "unset";
description = "unset";
descriptionClass = "noun";
check = value: true;
};
unset = {
_type = "unset";
};
isUnset = isType "unset";
unsetOr = typ:
(types.either unsetType typ)
// {
inherit (typ) description getSubOptions;
};
mkUnsetOption = opts:
mkOption (opts
// {
type = unsetOr opts.type;
default = opts.default or unset;
defaultText = literalExpression "unset";
});
filterUnset = value:
if builtins.isAttrs value && !builtins.hasAttr "_type" value
then let
filteredAttrs = builtins.mapAttrs (n: v: filterUnset v) value;
in
filterAttrs (name: value: (!isUnset value)) filteredAttrs
else if builtins.isList value
then builtins.filter (elem: !isUnset elem) (map filterUnset value)
else value;
testsSubmodule = {
config,
testsBase,
pos,
...
}: {
options = {
pos = mkUnsetOption {
type = types.attrs;
description = ''
Position of test, use `__curPos` for automatic insertion of current position.
'';
default = pos;
apply = val:
if isUnset val
then val
else let
fileRelative = removePrefix testsBase val.file;
in "${fileRelative}:${toString val.line}";
};
type = mkOption {
type = types.enum ["unit" "snapshot" "script"];
description = ''
Type of test, has to be one of "unit", "snapshot" or "script".
'';
default = "unit";
apply = value:
assert assertMsg (value != "script" || !isUnset config.script)
"test '${config.name}' as type 'script' requires 'script' to be set";
assert assertMsg (value != "unit" || !isUnset config.expected)
"test '${config.name}' as type 'unit' requires 'expected' to be set";
assert assertMsg (
let
actualIsUnset = isUnset config.actual;
actualDrvIsUnset = isUnset config.actualDrv;
in
(value != "unit")
|| (!actualIsUnset && actualDrvIsUnset)
|| (actualIsUnset && !actualDrvIsUnset)
)
"test '${config.name}' as type 'unit' requires only 'actual' OR 'actualDrv' to be set"; value;
};
name = mkOption {
type = types.str;
description = ''
Name of this test.
'';
};
description = mkUnsetOption {
type = types.str;
description = ''
Short description of the test.
'';
};
format = mkOption {
type = types.enum ["json" "pretty"];
description = ''
Which format to use for serializing arbitrary values.
Required since this config is serialized to JSON for passing it to Nixtest, so no Nix-values can be used directly.
- `json`: serializes the data to json using `builtins.toJSON`
- `pretty`: serializes the data to a "pretty" format using `lib.generators.toPretty`
'';
default = "json";
};
expected = mkUnsetOption {
type = types.anything;
description = ''
Expected value of the test. Remember, the values are serialized (see [here](#suitesnametestsformat)).
'';
apply = val:
if isUnset val || config.format == "json"
then val
else generators.toPretty {} val;
};
actual = mkUnsetOption {
type = types.anything;
description = ''
Actual value of the test. Remember, the values are serialized (see [here](#suitesnametestsformat)).
'';
apply = val:
if isUnset val || config.format == "json"
then val
else generators.toPretty {} val;
};
actualDrv = mkUnsetOption {
type = types.package;
description = ''
Actual value of the test, but as a derivation.
Nixtest will build this derivation when running the test, then compare the contents of the
resulting file to the [`expected`](#suitesnametestsexpected) value.
'';
apply = val:
# keep unset value
if isUnset val
then val
else builtins.unsafeDiscardStringContext (val.drvPath or "");
};
script = mkUnsetOption {
type = types.str;
description = ''
Script to run for the test.
Nixtest will run this, failing the test if it exits with a non-zero exit code.
'';
apply = val:
if isUnset val
then val
else
builtins.unsafeDiscardStringContext
(pkgs.writeShellScript "nixtest-${config.name}" val).drvPath;
};
};
};
suitesSubmodule = {
name,
config,
testsBase,
...
}: {
options = {
name = mkOption {
type = types.str;
description = ''
Name of the suite, uses attrset name by default.
'';
default = name;
defaultText = literalExpression name;
};
pos = mkUnsetOption {
type = types.attrs;
description = ''
Position for tests, use `__curPos` for automatic insertion of current position.
This will set `pos` for every test of this suite, useful if the suite's tests are all in a single file.
'';
example = literalExpression "__curPos";
};
tests = mkOption {
type = types.listOf (types.submoduleWith {
modules = [testsSubmodule];
specialArgs = {
inherit (config) pos;
inherit testsBase;
};
});
description = ''
Define tests of this suite here.
'';
default = [];
};
};
};
nixtestSubmodule = {config, ...}: {
_file = ./module.nix;
options = {
base = mkOption {
type = types.str;
description = ''
Base directory of the tests, will be removed from the test file path.
This makes it possible to show the relative path from the git repo, instead of ugly Nix store paths.
'';
default = "";
};
skip = mkOption {
type = types.str;
description = ''
Tests to skip, is passed to Nixtest's `--skip` param.
'';
default = "";
};
suites = mkOption {
type = types.attrsOf (types.submoduleWith {
modules = [suitesSubmodule];
specialArgs = {
testsBase = config.base;
};
});
description = ''
Define your test suites here, every test belongs to a suite.
'';
default = {};
apply = suites:
map (
n: filterUnset (builtins.removeAttrs suites.${n} ["pos"])
)
(builtins.attrNames suites);
example = {
"Suite A".tests = [
{
name = "Some Test";
}
];
};
};
finalConfigJson = mkOption {
internal = true;
type = types.package;
};
app = mkOption {
internal = true;
type = types.package;
};
};
config = {
finalConfigJson = nixtest-lib.exportSuites config.suites;
app = nixtest-lib.mkBinary {
nixtests = config.finalConfigJson;
extraParams = ''--skip="${config.skip}"'';
};
};
};
in
nixtestSubmodule

View file

@ -1,41 +0,0 @@
output=
exit_code=
function assert() {
test $1 || { echo "Assertion '$1' failed: $2" >&2; exit 1; }
}
function assert_eq() {
assert "$1 -eq $2" "$3"
}
function assert_not_eq() {
assert "$1 -ne $2" "$3"
}
function assert_contains() {
echo "$1" | grep -q -- "$2" || {
echo "Assertion failed: $3. $1 does not contain $2" >&2;
exit 1;
}
}
function assert_not_contains() {
echo "$1" | grep -q -- "$2" && {
echo "Assertion failed: $3. $1 does contain $2" >&2;
exit 1;
}
}
function assert_file_contains() {
grep -q -- "$2" $1 || {
echo "Assertion failed: $3. $1 does not contain $2" >&2;
exit 1;
}
}
function assert_file_not_contains() {
grep -q -- "$2" $1 && {
echo "Assertion failed: $3. $1 does contain $2" >&2;
exit 1;
}
}
function run() {
output=$($@ 2>&1)
exit_code=$?
}

View file

@ -1,7 +0,0 @@
{lib, ...}: {
path = pkgs: "export PATH=${lib.makeBinPath pkgs}";
pathAdd = pkgs: "export PATH=$PATH:${lib.makeBinPath pkgs}";
scriptHelpers = builtins.readFile ./scriptHelpers.sh;
toJsonFile = any: builtins.toFile "actual" (builtins.unsafeDiscardStringContext (builtins.toJSON any));
toPrettyFile = any: builtins.toFile "actual" (lib.generators.toPretty {} any);
}

View file

@ -1,97 +0,0 @@
{
lib,
pkgs,
...
}: {
skip = "skip.*d";
suites = {
"suite-one" = {
# required to figure out file and line, but optional
pos = __curPos;
tests = [
{
name = "test-one";
expected = 1;
actual = 1;
}
{
name = "fail";
expected = 0;
actual = "meow";
}
{
name = "snapshot-test";
type = "snapshot";
actual = "test";
}
{
name = "test-snapshot-drv";
type = "snapshot";
actualDrv = pkgs.runCommand "test-snapshot" {} ''
echo '"snapshot drv"' > $out
'';
}
{
name = "test-error-drv";
expected = null;
actualDrv = pkgs.runCommand "test-error-drv" {} ''
echo "This works, but its better to just write 'fail' to \$out and expect 'success' or sth."
exit 1
'';
}
{
name = "test-script";
type = "script";
script = ''
echo Test something here
# required in pure mode:
export PATH="${lib.makeBinPath [pkgs.gnugrep]}"
grep -q "test" ${builtins.toFile "test" "test"}
'';
}
];
};
"other-suite".tests = [
{
name = "obj-snapshot";
type = "snapshot";
pos = __curPos;
actual = {hello = "world";};
}
{
name = "pretty-snapshot";
type = "snapshot";
format = "pretty";
pos = __curPos;
actual = {
example = args: {};
example2 = {
drv = pkgs.hello;
};
};
}
{
name = "pretty-unit";
format = "pretty";
pos = __curPos;
expected = pkgs.hello;
actual = pkgs.hello;
}
{
name = "test-drv";
pos = __curPos;
expected = {a = "b";};
actualDrv = pkgs.runCommand "test-something" {} ''
echo "Simulating taking some time"
sleep 1
echo '{"a":"b"}' > $out
'';
}
{
name = "skipped";
expected = null;
actual = null;
}
];
};
}

View file

@ -1,99 +0,0 @@
{
pkgs,
ntlib,
...
}: {
suites."Lib Tests" = {
pos = __curPos;
tests = [
{
name = "autodiscovery";
type = "script";
script = let
actual = ntlib.helpers.toPrettyFile (ntlib.autodiscover {
dir = ./fixtures;
});
# tests if strings with store path context work
actualDirString = ntlib.helpers.toPrettyFile (ntlib.autodiscover {
dir = "${./fixtures}";
});
in
# sh
''
${ntlib.helpers.path [pkgs.gnugrep]}
${ntlib.helpers.scriptHelpers}
assert_file_contains ${actual} "sample_test.nix" "should find sample_test.nix"
assert_file_contains ${actual} "base = \"/nix/store/.*-source/tests/fixtures/\"" "should set base to fixtures dir"
assert_file_contains ${actualDirString} "sample_test.nix" "should find sample_test.nix"
assert_file_contains ${actualDirString} "base = \"/nix/store/.*-fixtures/\"" "should set base to fixtures dir"
'';
}
{
name = "binary";
type = "script";
script = let
binary =
(ntlib.mkBinary {
nixtests = "stub";
extraParams = "--impure";
})
+ "/bin/nixtests:run";
in
# sh
''
${ntlib.helpers.path [pkgs.gnugrep]}
${ntlib.helpers.scriptHelpers}
assert_file_contains ${binary} "nixtest" "should contain nixtest"
assert_file_contains ${binary} "--impure" "should contain --impure arg"
assert_file_contains ${binary} "--tests=stub" "should contain --tests arg"
run "${binary} --help"
assert_eq $exit_code 0 "should exit 0"
assert_contains "$output" "Usage of nixtest" "should show help"
run "${binary}"
assert_eq $exit_code 1 "should exit 1"
assert_contains "$output" "Tests file does not exist"
'';
}
{
name = "full run with fixtures";
type = "script";
script = let
binary =
(ntlib.mkNixtest {
modules = ntlib.autodiscover {dir = ./fixtures;};
args = {inherit pkgs;};
})
+ "/bin/nixtests:run";
in
# sh
''
${ntlib.helpers.path [pkgs.gnugrep pkgs.mktemp pkgs.coreutils]}
${ntlib.helpers.scriptHelpers}
export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
export NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
cp -r ${./../snapshots} snapshots
# start without nix & env binaries to expect errors
run "${binary} --junit=junit.xml"
assert "$exit_code -eq 2" "should exit 2"
assert "-f junit.xml" "should create junit.xml"
assert_contains "$output" "executable file not found" "nix should not be found in pure mode"
# now add required deps
${ntlib.helpers.pathAdd [pkgs.nix pkgs.coreutils]}
run "${binary} --junit=junit2.xml"
assert "$exit_code -eq 2" "should exit 2"
assert "-f junit2.xml" "should create junit2.xml"
assert_not_contains "$output" "executable file not found" "nix should now exist"
assert_contains "$output" "suite-one" "should contain suite-one"
assert_contains "$output" "8/11 (1 SKIPPED)" "should be 8/11 total"
assert_contains "$output" "ERROR" "should contain an error"
assert_contains "$output" "SKIP" "should contain a skip"
'';
}
];
};
}