mirror of
https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git
synced 2025-12-12 02:00:13 +01:00
Merge branch 'testing' into 'main'
Draft: Tests See merge request TECHNOFAB/nix-gitlab-ci!12
This commit is contained in:
commit
5ff1f16c82
20 changed files with 733 additions and 288 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -3,3 +3,4 @@
|
|||
.direnv
|
||||
.pre-commit-config.yaml
|
||||
result
|
||||
*.xml
|
||||
|
|
|
|||
19
flake.lock
generated
19
flake.lock
generated
|
|
@ -329,6 +329,24 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixtest": {
|
||||
"locked": {
|
||||
"dir": "lib",
|
||||
"lastModified": 1748945995,
|
||||
"narHash": "sha256-OWflapMq78nCUT7N0vS/AuK4E8y7p8oh0zjnDioJ4Lw=",
|
||||
"owner": "technofab",
|
||||
"repo": "nixtest",
|
||||
"rev": "392e8796f386373a36aecd3216925accf2714a1a",
|
||||
"type": "gitlab"
|
||||
},
|
||||
"original": {
|
||||
"dir": "lib",
|
||||
"owner": "technofab",
|
||||
"ref": "1.0.0",
|
||||
"repo": "nixtest",
|
||||
"type": "gitlab"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"devenv": "devenv",
|
||||
|
|
@ -336,6 +354,7 @@
|
|||
"mkdocs-material-umami": "mkdocs-material-umami",
|
||||
"nix-mkdocs": "nix-mkdocs",
|
||||
"nixpkgs": "nixpkgs_4",
|
||||
"nixtest": "nixtest",
|
||||
"systems": "systems",
|
||||
"treefmt-nix": "treefmt-nix"
|
||||
}
|
||||
|
|
|
|||
19
flake.nix
19
flake.nix
|
|
@ -9,6 +9,7 @@
|
|||
inputs.devenv.flakeModule
|
||||
inputs.treefmt-nix.flakeModule
|
||||
inputs.nix-mkdocs.flakeModule
|
||||
inputs.nixtest.flakeModule
|
||||
./lib/flakeModule.nix
|
||||
];
|
||||
systems = import systems;
|
||||
|
|
@ -19,6 +20,9 @@
|
|||
system,
|
||||
...
|
||||
}: rec {
|
||||
imports = [
|
||||
./tests
|
||||
];
|
||||
treefmt = {
|
||||
projectRootFile = "flake.nix";
|
||||
programs = {
|
||||
|
|
@ -139,7 +143,7 @@
|
|||
};
|
||||
# should set the "default" pipeline
|
||||
ci = {
|
||||
stages = ["test" "build" "deploy"];
|
||||
stages = ["test" "nixtest" "build" "deploy"];
|
||||
jobs = {
|
||||
"test" = {
|
||||
stage = "test";
|
||||
|
|
@ -195,6 +199,18 @@
|
|||
}
|
||||
];
|
||||
};
|
||||
"nixtest" = {
|
||||
stage = "nixtest";
|
||||
script = [
|
||||
# sh
|
||||
"nix run .#nixtests:run -- --junit=junit.xml --pure"
|
||||
];
|
||||
allow_failure = true;
|
||||
artifacts = {
|
||||
when = "always";
|
||||
reports.junit = "junit.xml";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
pipelines."non-default" = {
|
||||
|
|
@ -269,6 +285,7 @@
|
|||
treefmt-nix.url = "github:numtide/treefmt-nix";
|
||||
nix-mkdocs.url = "gitlab:technofab/nixmkdocs?dir=lib";
|
||||
mkdocs-material-umami.url = "gitlab:technofab/mkdocs-material-umami";
|
||||
nixtest.url = "gitlab:technofab/nixtest/1.0.0?dir=lib";
|
||||
};
|
||||
|
||||
nixConfig = {
|
||||
|
|
|
|||
8
lib/default.nix
Normal file
8
lib/default.nix
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
args: {
|
||||
helpers = import ./helpers.nix args;
|
||||
jobDeps = import ./jobDeps.nix args;
|
||||
jobRun = import ./jobRun.nix args;
|
||||
jobPatch = import ./jobPatch.nix args;
|
||||
pipeline = import ./pipeline.nix args;
|
||||
utils = import ./utils.nix args;
|
||||
}
|
||||
|
|
@ -1,9 +1,8 @@
|
|||
{
|
||||
description = "Nix-CI lib";
|
||||
description = "Nix-GitLab-CI lib";
|
||||
|
||||
outputs = {...} @ inputs:
|
||||
{
|
||||
outputs = {...}: {
|
||||
flakeModule = import ./flakeModule.nix;
|
||||
}
|
||||
// (import ./utils.nix);
|
||||
lib = import ./default.nix;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,22 +9,12 @@
|
|||
pkgs,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList;
|
||||
cilib = import ./. {inherit lib pkgs;};
|
||||
inherit (cilib.pipeline) mkPipeline;
|
||||
inherit (lib) types mkOption;
|
||||
|
||||
cfg = config.ci.config;
|
||||
|
||||
stdenvMinimal = pkgs.stdenvNoCC.override {
|
||||
cc = null;
|
||||
preHook = "";
|
||||
allowedRequisites = null;
|
||||
initialPath = [pkgs.coreutils pkgs.findutils];
|
||||
extraNativeBuildInputs = [];
|
||||
};
|
||||
|
||||
filterAttrsRec = pred: v:
|
||||
if isAttrs v
|
||||
then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v)
|
||||
else v;
|
||||
|
||||
subType = options: types.submodule {inherit options;};
|
||||
mkNullOption = type:
|
||||
mkOption {
|
||||
|
|
@ -168,228 +158,15 @@
|
|||
};
|
||||
};
|
||||
|
||||
config.legacyPackages = let
|
||||
# NOTE: json is also valid yaml and this removes dependency on jq
|
||||
# and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml
|
||||
# respectively)
|
||||
toYaml = name: value: builtins.toFile name (builtins.toJSON value);
|
||||
mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
|
||||
prepend = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
${key} =
|
||||
arr
|
||||
++ (job.${key} or []);
|
||||
};
|
||||
append = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
${key} = (job.${key} or []) ++ arr;
|
||||
};
|
||||
prependToBeforeScript = prepend "before_script";
|
||||
appendToAfterScript = append "after_script";
|
||||
|
||||
# filter job's variables to either only those containing store paths
|
||||
# or those that do not
|
||||
filterJobVariables = nix: job:
|
||||
lib.concatMapAttrs (
|
||||
name: value:
|
||||
lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) {
|
||||
${name} = value;
|
||||
}
|
||||
)
|
||||
(job.variables or {});
|
||||
in
|
||||
lib.fold (pipeline: acc: acc // pipeline) {} (map (
|
||||
pipeline_name: let
|
||||
config.legacyPackages = lib.fold (pipeline: acc: acc // pipeline) {} (
|
||||
map (
|
||||
pipeline_name:
|
||||
(mkPipeline {
|
||||
pipeline = config.pipelines."${pipeline_name}";
|
||||
jobs = filterAttrsRec (n: v: v != null) pipeline.jobs;
|
||||
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]);
|
||||
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
|
||||
# pro: we don't have to download everything, just the deps for the current job
|
||||
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
|
||||
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
|
||||
jobsMappedForDeps =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithStorePaths = filterJobVariables true job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
||||
name = pipeline_name;
|
||||
}).packages
|
||||
) (builtins.attrNames config.pipelines)
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}";
|
||||
value = stdenvMinimal.mkDerivation {
|
||||
name = "gitlab-ci-job-deps-${key}";
|
||||
dontUnpack = true;
|
||||
installPhase = let
|
||||
script = ''
|
||||
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
|
||||
# variables containing nix derivations:
|
||||
${variableExports}
|
||||
'';
|
||||
in
|
||||
# sh
|
||||
''
|
||||
echo '${script}' > $out
|
||||
chmod +x $out
|
||||
'';
|
||||
};
|
||||
})
|
||||
jobs;
|
||||
# allows the user to directly run the script
|
||||
jobsMappedForScript =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithoutStorePaths = filterJobVariables false job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}";
|
||||
value = let
|
||||
actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" ''
|
||||
# set up deps and environment variables containing store paths
|
||||
. ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"}
|
||||
# normal environment variables
|
||||
${variableExports}
|
||||
# run before_script, script and after_script
|
||||
echo -e "\e[32mRunning before_script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines (job.before_script or [])}
|
||||
{ set +x; } 2>/dev/null
|
||||
echo -e "\e[32mRunning script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines job.script}
|
||||
{ set +x; } 2>/dev/null
|
||||
echo -e "\e[32mRunning after_script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines (job.after_script or [])}
|
||||
{ set +x; } 2>/dev/null
|
||||
'';
|
||||
sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" ''
|
||||
echo -e "\e[32mSetting up...\e[0m"
|
||||
|
||||
actualJobScript=$1
|
||||
shift
|
||||
|
||||
INCLUDE_DIRTY=false
|
||||
NO_SANDBOX=false
|
||||
KEEP_TMP=false
|
||||
KEEP_ENV=""
|
||||
# parse flags
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--include-dirty)
|
||||
INCLUDE_DIRTY=true
|
||||
shift
|
||||
;;
|
||||
--no-sandbox)
|
||||
NO_SANDBOX=true
|
||||
shift
|
||||
;;
|
||||
--keep-tmp)
|
||||
KEEP_TMP=true
|
||||
shift
|
||||
;;
|
||||
--keep-env)
|
||||
KEEP_ENV="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$NO_SANDBOX" = false ]; then
|
||||
echo "Running with simple sandboxing"
|
||||
if [ "$KEEP_TMP" = false ]; then
|
||||
trap "rm -rf '$TMPDIR'" EXIT
|
||||
else
|
||||
echo "Temp dir will be preserved at: $TMPDIR"
|
||||
fi
|
||||
|
||||
# check if dirty
|
||||
DIRTY_PATCH=""
|
||||
if ! git diff --quiet && ! git diff --staged --quiet; then
|
||||
echo "Warning: working tree is dirty."
|
||||
DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch")
|
||||
git diff --staged > "$DIRTY_PATCH"
|
||||
trap "rm -f '$DIRTY_PATCH'" EXIT
|
||||
fi
|
||||
TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX")
|
||||
git clone . $TMPDIR
|
||||
pushd $TMPDIR >/dev/null
|
||||
if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then
|
||||
echo "Copying dirty changes..."
|
||||
git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes"
|
||||
fi
|
||||
|
||||
echo "Running job in $TMPDIR"
|
||||
env -i $(
|
||||
if [[ -n "$KEEP_ENV" ]]; then
|
||||
IFS=',' read -ra VARS <<< "$KEEP_ENV"
|
||||
for var in "''${VARS[@]}"; do
|
||||
printf '%s=%q ' "$var" "''${!var}"
|
||||
done
|
||||
fi
|
||||
) bash $actualJobScript
|
||||
popd >/dev/null
|
||||
else
|
||||
exec $actualJobScript
|
||||
fi
|
||||
'';
|
||||
in
|
||||
# this way the sandbox helper just needs to be built once
|
||||
pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
|
||||
exec ${lib.getExe sandboxHelper} ${actualJobScript} $@
|
||||
'';
|
||||
})
|
||||
jobs;
|
||||
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
|
||||
jobsPatched =
|
||||
mapAttrs (key: job: {
|
||||
name = key;
|
||||
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
|
||||
builtins.removeAttrs (
|
||||
(prependToBeforeScript [
|
||||
"source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\""
|
||||
]
|
||||
(appendToAfterScript [
|
||||
"finalize_nix_ci"
|
||||
]
|
||||
job))
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
image = job.image;
|
||||
variables =
|
||||
(filterJobVariables false job)
|
||||
// lib.optionalAttrs job.nix.enable-runner-cache {
|
||||
NIX_CI_CACHE_STRATEGY = "runner";
|
||||
};
|
||||
cache =
|
||||
(
|
||||
let
|
||||
c = job.cache or [];
|
||||
in
|
||||
toList c
|
||||
)
|
||||
++ (lib.optional (job.nix.enable-runner-cache) {
|
||||
key = job.nix.runner-cache-key;
|
||||
paths = [".nix-cache/"];
|
||||
});
|
||||
}
|
||||
) ["nix"];
|
||||
})
|
||||
jobs;
|
||||
in
|
||||
# gitlab-ci:pipeline:<name>
|
||||
# gitlab-ci:pipeline:<name>:job:<name>
|
||||
# gitlab-ci:pipeline:<name>:job-deps:<name>
|
||||
{
|
||||
"gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched);
|
||||
}
|
||||
// jobsMappedForDeps
|
||||
// jobsMappedForScript
|
||||
) (builtins.attrNames config.pipelines));
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
|
|||
49
lib/helpers.nix
Normal file
49
lib/helpers.nix
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
{lib, ...} @ args: let
|
||||
inherit (lib) isAttrs filterAttrs mapAttrs;
|
||||
in rec {
|
||||
prepend = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs (job.nix.enable or false) {
|
||||
${key} =
|
||||
arr
|
||||
++ (job.${key} or []);
|
||||
};
|
||||
append = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs (job.nix.enable or false) {
|
||||
${key} = (job.${key} or []) ++ arr;
|
||||
};
|
||||
prependToBeforeScript = prepend "before_script";
|
||||
appendToAfterScript = append "after_script";
|
||||
|
||||
# json is also valid yaml and this removes dependency on jq and/or remarshal
|
||||
# (used in pkgs.formats.json and pkgs.formats.yaml respectively)
|
||||
toYaml = name: value: builtins.toFile name (builtins.toJSON value);
|
||||
|
||||
customMapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
|
||||
|
||||
filterAttrsRec = pred: v:
|
||||
if isAttrs v
|
||||
then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v)
|
||||
else v;
|
||||
|
||||
# filter job's variables to either only those containing store paths
|
||||
# or those that do not
|
||||
filterJobVariables = nix: job:
|
||||
lib.concatMapAttrs (
|
||||
name: value:
|
||||
lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) {
|
||||
${name} = value;
|
||||
}
|
||||
)
|
||||
(job.variables or {});
|
||||
|
||||
# args.pkgs so "pkgs" does not need to be passed all the time
|
||||
stdenvMinimal = args.pkgs.stdenvNoCC.override {
|
||||
cc = null;
|
||||
preHook = "";
|
||||
allowedRequisites = null;
|
||||
initialPath = with args.pkgs; [coreutils findutils];
|
||||
extraNativeBuildInputs = [];
|
||||
};
|
||||
}
|
||||
36
lib/jobDeps.nix
Normal file
36
lib/jobDeps.nix
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./. {inherit lib pkgs;};
|
||||
inherit (cilib.helpers) filterJobVariables stdenvMinimal;
|
||||
in {
|
||||
mkJobDeps = {
|
||||
key,
|
||||
job,
|
||||
}: let
|
||||
variablesWithStorePaths = filterJobVariables true job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
||||
);
|
||||
script = ''
|
||||
export PATH="${lib.makeBinPath (job.nix.deps or [])}:$PATH";
|
||||
# variables containing nix derivations:
|
||||
${variableExports}
|
||||
'';
|
||||
in
|
||||
stdenvMinimal.mkDerivation {
|
||||
name = "gitlab-ci-job-deps-${key}";
|
||||
dontUnpack = true;
|
||||
installPhase =
|
||||
# sh
|
||||
''
|
||||
echo '${script}' > $out
|
||||
chmod +x $out
|
||||
'';
|
||||
passthru = {
|
||||
inherit script;
|
||||
};
|
||||
};
|
||||
}
|
||||
49
lib/jobPatch.nix
Normal file
49
lib/jobPatch.nix
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./. {inherit lib pkgs;};
|
||||
inherit (lib) toList;
|
||||
inherit (cilib.helpers) prependToBeforeScript appendToAfterScript filterJobVariables;
|
||||
in {
|
||||
mkJobPatched = {
|
||||
key,
|
||||
job,
|
||||
pipeline_name,
|
||||
}:
|
||||
builtins.removeAttrs (
|
||||
(prependToBeforeScript [
|
||||
"source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\""
|
||||
]
|
||||
(appendToAfterScript [
|
||||
"finalize_nix_ci"
|
||||
]
|
||||
job))
|
||||
// lib.optionalAttrs job.nix.enable (
|
||||
(let
|
||||
variables =
|
||||
(filterJobVariables false job)
|
||||
// lib.optionalAttrs job.nix.enable-runner-cache {
|
||||
NIX_CI_CACHE_STRATEGY = "runner";
|
||||
};
|
||||
in
|
||||
# filter empty variables
|
||||
lib.optionalAttrs (variables != {}) {
|
||||
inherit variables;
|
||||
})
|
||||
// (let
|
||||
cache =
|
||||
(toList (job.cache or []))
|
||||
++ (lib.optional (job.nix.enable-runner-cache) {
|
||||
key = job.nix.runner-cache-key;
|
||||
paths = [".nix-cache/"];
|
||||
});
|
||||
in
|
||||
# filter empty cache
|
||||
lib.optionalAttrs (cache != []) {
|
||||
inherit cache;
|
||||
})
|
||||
)
|
||||
) ["nix"];
|
||||
}
|
||||
48
lib/jobRun.nix
Normal file
48
lib/jobRun.nix
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./. {inherit lib pkgs;};
|
||||
inherit (cilib.helpers) filterJobVariables;
|
||||
in {
|
||||
mkJobRun = {
|
||||
key,
|
||||
job,
|
||||
jobDeps,
|
||||
}: let
|
||||
variablesWithoutStorePaths = filterJobVariables false job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
|
||||
);
|
||||
sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" (builtins.readFile ./sandbox_helper.sh);
|
||||
actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" ''
|
||||
# set up deps and environment variables containing store paths
|
||||
. ${jobDeps}
|
||||
# normal environment variables
|
||||
${variableExports}
|
||||
# run before_script, script and after_script
|
||||
echo -e "\e[32mRunning before_script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines (job.before_script or [])}
|
||||
{ set +x; } 2>/dev/null
|
||||
echo -e "\e[32mRunning script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines job.script}
|
||||
{ set +x; } 2>/dev/null
|
||||
echo -e "\e[32mRunning after_script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines (job.after_script or [])}
|
||||
{ set +x; } 2>/dev/null
|
||||
'';
|
||||
in
|
||||
# this way the sandbox helper just needs to be built once
|
||||
pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
|
||||
exec ${lib.getExe sandboxHelper} ${actualJobScript} $@
|
||||
''
|
||||
// {
|
||||
passthru = {
|
||||
inherit jobDeps actualJobScript;
|
||||
};
|
||||
};
|
||||
}
|
||||
61
lib/pipeline.nix
Normal file
61
lib/pipeline.nix
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./. {inherit lib pkgs;};
|
||||
inherit (cilib.helpers) filterAttrsRec customMapAttrs toYaml;
|
||||
inherit (cilib.jobDeps) mkJobDeps;
|
||||
inherit (cilib.jobRun) mkJobRun;
|
||||
inherit (cilib.jobPatch) mkJobPatched;
|
||||
in {
|
||||
mkPipeline = {
|
||||
name,
|
||||
pipeline,
|
||||
}: let
|
||||
jobs = filterAttrsRec (n: v: v != null) pipeline.jobs;
|
||||
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]);
|
||||
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
|
||||
# pro: we don't have to download everything, just the deps for the current job
|
||||
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
|
||||
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
|
||||
jobsMappedForDeps =
|
||||
customMapAttrs (key: job: {
|
||||
name = "gitlab-ci:pipeline:${name}:job-deps:${key}";
|
||||
value = mkJobDeps {inherit key job;};
|
||||
})
|
||||
jobs;
|
||||
# allows the user to directly run the script
|
||||
jobsMappedForScript =
|
||||
customMapAttrs (key: job: {
|
||||
name = "gitlab-ci:pipeline:${name}:job:${key}";
|
||||
value = mkJobRun {
|
||||
inherit key job;
|
||||
jobDeps = jobsMappedForDeps."gitlab-ci:pipeline:${name}:job-deps:${key}";
|
||||
};
|
||||
})
|
||||
jobs;
|
||||
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
|
||||
jobsPatched =
|
||||
customMapAttrs (key: job: {
|
||||
name = key;
|
||||
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
|
||||
mkJobPatched {
|
||||
inherit key job;
|
||||
pipeline_name = name;
|
||||
};
|
||||
})
|
||||
jobs;
|
||||
in {
|
||||
packages =
|
||||
# gitlab-ci:pipeline:<name>
|
||||
# gitlab-ci:pipeline:<name>:job:<name>
|
||||
# gitlab-ci:pipeline:<name>:job-deps:<name>
|
||||
{
|
||||
"gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-${name}.yml" (rest // jobsPatched);
|
||||
}
|
||||
// jobsMappedForDeps
|
||||
// jobsMappedForScript;
|
||||
finalConfig = rest // jobsPatched;
|
||||
};
|
||||
}
|
||||
73
lib/sandbox_helper.sh
Normal file
73
lib/sandbox_helper.sh
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
echo -e "\e[32mSetting up...\e[0m"
|
||||
|
||||
actualJobScript=$1
|
||||
shift
|
||||
|
||||
INCLUDE_DIRTY=false
|
||||
NO_SANDBOX=false
|
||||
KEEP_TMP=false
|
||||
KEEP_ENV=""
|
||||
# parse flags
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--include-dirty)
|
||||
INCLUDE_DIRTY=true
|
||||
shift
|
||||
;;
|
||||
--no-sandbox)
|
||||
NO_SANDBOX=true
|
||||
shift
|
||||
;;
|
||||
--keep-tmp)
|
||||
KEEP_TMP=true
|
||||
shift
|
||||
;;
|
||||
--keep-env)
|
||||
KEEP_ENV="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$NO_SANDBOX" = false ]; then
|
||||
echo "Running with simple sandboxing"
|
||||
if [ "$KEEP_TMP" = false ]; then
|
||||
trap "rm -rf '$TMPDIR'" EXIT
|
||||
else
|
||||
echo "Temp dir will be preserved at: $TMPDIR"
|
||||
fi
|
||||
|
||||
# check if dirty
|
||||
DIRTY_PATCH=""
|
||||
if ! git diff --quiet && ! git diff --staged --quiet; then
|
||||
echo "Warning: working tree is dirty."
|
||||
DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch")
|
||||
git diff --staged > "$DIRTY_PATCH"
|
||||
trap "rm -f '$DIRTY_PATCH'" EXIT
|
||||
fi
|
||||
TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX")
|
||||
git clone . $TMPDIR
|
||||
pushd $TMPDIR >/dev/null
|
||||
if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then
|
||||
echo "Copying dirty changes..."
|
||||
git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes"
|
||||
fi
|
||||
|
||||
echo "Running job in $TMPDIR"
|
||||
env -i $(
|
||||
if [[ -n "$KEEP_ENV" ]]; then
|
||||
IFS=',' read -ra VARS <<< "$KEEP_ENV"
|
||||
for var in "${VARS[@]}"; do
|
||||
printf '%s=%q ' "$var" "${!var}"
|
||||
done
|
||||
fi
|
||||
) bash $actualJobScript
|
||||
popd >/dev/null
|
||||
else
|
||||
exec $actualJobScript
|
||||
fi
|
||||
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
{
|
||||
mkUtils = {pkgs, ...}: {
|
||||
{pkgs, ...}: {
|
||||
commitAndPushFiles = {
|
||||
message,
|
||||
files ? [],
|
||||
|
|
@ -9,6 +8,7 @@
|
|||
before_script =
|
||||
(jobArgs.before_script or [])
|
||||
++ [
|
||||
# sh
|
||||
''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles"
|
||||
eval "$(ssh-agent -s)" >/dev/null;
|
||||
|
|
@ -30,6 +30,7 @@
|
|||
in
|
||||
(jobArgs.script or [])
|
||||
++ [
|
||||
# sh
|
||||
''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary"
|
||||
${addScript}
|
||||
|
|
@ -42,5 +43,4 @@
|
|||
];
|
||||
nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
1
snapshots/default.snap.json
Normal file
1
snapshots/default.snap.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"docs":{"after_script":["finalize_nix_ci"],"artifacts":{"paths":["public"]},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:docs\""],"image":"$NIX_CI_IMAGE","script":["nix build .#docs:default\nmkdir -p public\ncp -r result/. public/\n"],"stage":"build"},"nixtest":{"after_script":["finalize_nix_ci"],"allow_failure":true,"artifacts":{"reports":{"junit":"junit.xml"},"when":"always"},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:nixtest\""],"image":"$NIX_CI_IMAGE","script":["nix run .#nixtests:run -- --junit=junit.xml --pure"],"stage":"nixtest"},"pages":{"artifacts":{"paths":["public"]},"image":"alpine:latest","rules":[{"if":"$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"}],"script":["true"],"stage":"deploy"},"stages":["test","nixtest","build","deploy"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test\""],"cache":[{"key":"$CI_JOB_NAME-$CI_COMMIT_REF_SLUG","paths":[".nix-cache/"]}],"image":"$NIX_CI_IMAGE","script":["hello","curl google.de","echo $TEST $TEST_WITH_DERIVATION"],"stage":"test","variables":{"NIX_CI_CACHE_STRATEGY":"runner","TEST":"test"}},"test-default":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test-default\""],"image":"$NIX_CI_IMAGE","script":["hello"],"stage":"test"},"test-non-nix":{"image":"alpine:latest","script":["echo \"This job will not be modified to use nix\""],"stage":"test"}}
|
||||
1
snapshots/non-default.snap.json
Normal file
1
snapshots/non-default.snap.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"stages":["test"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:non-default:job-deps:test\""],"image":"$NIX_CI_IMAGE","script":["echo Hello from another pipeline"],"stage":"test"}}
|
||||
140
tests/ci-lib.nix
Normal file
140
tests/ci-lib.nix
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./../lib {inherit lib pkgs;};
|
||||
in {
|
||||
nixtest.suites."CI Lib" = {
|
||||
pos = __curPos;
|
||||
tests = let
|
||||
inherit (cilib.jobDeps) mkJobDeps;
|
||||
inherit (cilib.jobRun) mkJobRun;
|
||||
inherit (cilib.jobPatch) mkJobPatched;
|
||||
inherit (cilib.pipeline) mkPipeline;
|
||||
deps = mkJobDeps {
|
||||
key = "test";
|
||||
job = {
|
||||
nix.deps = [pkgs.hello];
|
||||
variables.TEST = "${pkgs.curl}";
|
||||
};
|
||||
};
|
||||
in [
|
||||
{
|
||||
name = "jobDeps";
|
||||
type = "script";
|
||||
script =
|
||||
# sh
|
||||
''
|
||||
export PATH=${lib.makeBinPath [pkgs.gnugrep]}
|
||||
grep -q "/nix/store" ${deps}
|
||||
grep -q 'hello/bin:$PATH' ${deps}
|
||||
grep -q "export TEST=" ${deps}
|
||||
grep -q "curl" ${deps}
|
||||
'';
|
||||
}
|
||||
{
|
||||
name = "jobRun";
|
||||
type = "script";
|
||||
script = let
|
||||
run = mkJobRun {
|
||||
key = "test";
|
||||
job.script = ["hello"];
|
||||
jobDeps = deps;
|
||||
};
|
||||
in
|
||||
# sh
|
||||
''
|
||||
export PATH=${lib.makeBinPath [pkgs.gnugrep]}
|
||||
grep -q "sandbox-helper" ${run}/bin/gitlab-ci-job:test
|
||||
grep -q "gitlab-ci-job-test-raw" ${run}/bin/gitlab-ci-job:test
|
||||
grep -q "gitlab-ci-job-deps-test" ${run.passthru.actualJobScript}
|
||||
grep -q "Running script..." ${run.passthru.actualJobScript}
|
||||
grep -q "hello" ${run.passthru.actualJobScript}
|
||||
'';
|
||||
}
|
||||
{
|
||||
name = "jobPatched nix disabled";
|
||||
expected = {};
|
||||
actual = mkJobPatched {
|
||||
key = "test";
|
||||
pipeline_name = "test";
|
||||
job.nix.enable = false;
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "jobPatched without runner cache";
|
||||
expected = {
|
||||
after_script = ["finalize_nix_ci"];
|
||||
before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
|
||||
};
|
||||
actual = mkJobPatched {
|
||||
key = "test";
|
||||
pipeline_name = "test";
|
||||
job.nix = {
|
||||
enable = true;
|
||||
enable-runner-cache = false;
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "jobPatched with runner cache";
|
||||
expected = {
|
||||
after_script = ["finalize_nix_ci"];
|
||||
before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
|
||||
cache = [
|
||||
{
|
||||
key = "test";
|
||||
paths = [".nix-cache/"];
|
||||
}
|
||||
];
|
||||
variables."NIX_CI_CACHE_STRATEGY" = "runner";
|
||||
};
|
||||
actual = mkJobPatched {
|
||||
key = "test";
|
||||
pipeline_name = "test";
|
||||
job.nix = {
|
||||
enable = true;
|
||||
enable-runner-cache = true;
|
||||
runner-cache-key = "test";
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "mkPipeline empty";
|
||||
expected = {};
|
||||
actual =
|
||||
(mkPipeline {
|
||||
name = "test";
|
||||
pipeline.jobs = {};
|
||||
}).finalConfig;
|
||||
}
|
||||
{
|
||||
name = "mkPipeline empty packages";
|
||||
type = "script";
|
||||
script = let
|
||||
pipeline = builtins.toFile "pipeline-test" (builtins.toJSON
|
||||
(mkPipeline {
|
||||
name = "test";
|
||||
pipeline.jobs = {};
|
||||
}).packages);
|
||||
in
|
||||
# sh
|
||||
''
|
||||
set -euo pipefail
|
||||
export PATH=${lib.makeBinPath [pkgs.jq pkgs.gnugrep pkgs.coreutils]}
|
||||
# single key
|
||||
jq 'keys | length == 1' "${pipeline}" | grep -q true
|
||||
# key is exactly "gitlab-ci:pipeline:test"
|
||||
jq -r 'keys[0]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test"
|
||||
# value contains "/nix/store/"
|
||||
jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "/nix/store/"
|
||||
# value contains "gitlab-ci-test.yml"
|
||||
jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "gitlab-ci-test.yml"
|
||||
# file only contains "{}"
|
||||
[[ "$(cat $(jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}"))" == "{}" ]]
|
||||
'';
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
8
tests/default.nix
Normal file
8
tests/default.nix
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
imports = [
|
||||
./utils.nix
|
||||
./ci-lib.nix
|
||||
./helpers.nix
|
||||
./pipeline-yamls.nix
|
||||
];
|
||||
}
|
||||
96
tests/helpers.nix
Normal file
96
tests/helpers.nix
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./../lib {inherit lib pkgs;};
|
||||
in {
|
||||
nixtest.suites."Helpers" = {
|
||||
pos = __curPos;
|
||||
tests = let
|
||||
inherit (cilib) helpers;
|
||||
in [
|
||||
{
|
||||
name = "appendToAfterScript nix disabled";
|
||||
expected = {};
|
||||
actual = helpers.appendToAfterScript [] {};
|
||||
}
|
||||
{
|
||||
name = "appendToAfterScript empty";
|
||||
expected = {
|
||||
nix.enable = true;
|
||||
after_script = [];
|
||||
};
|
||||
actual = helpers.appendToAfterScript [] {nix.enable = true;};
|
||||
}
|
||||
{
|
||||
name = "appendToAfterScript";
|
||||
expected = {
|
||||
nix.enable = true;
|
||||
after_script = ["echo after_script" "finalize_nix_ci"];
|
||||
};
|
||||
actual = helpers.appendToAfterScript ["finalize_nix_ci"] {
|
||||
nix.enable = true;
|
||||
after_script = ["echo after_script"];
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "prependToBeforeScript nix disabled";
|
||||
expected = {};
|
||||
actual = helpers.prependToBeforeScript [] {};
|
||||
}
|
||||
{
|
||||
name = "prependToBeforeScript empty";
|
||||
expected = {
|
||||
nix.enable = true;
|
||||
before_script = [];
|
||||
};
|
||||
actual = helpers.prependToBeforeScript [] {nix.enable = true;};
|
||||
}
|
||||
{
|
||||
name = "prependToBeforeScript";
|
||||
expected = {
|
||||
nix.enable = true;
|
||||
before_script = ["setup_nix_ci" "echo before_script"];
|
||||
};
|
||||
actual = helpers.prependToBeforeScript ["setup_nix_ci"] {
|
||||
nix.enable = true;
|
||||
before_script = ["echo before_script"];
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "toYaml";
|
||||
expected = ''{"hello":"world"}'';
|
||||
actual = builtins.readFile (helpers.toYaml "test" {hello = "world";});
|
||||
}
|
||||
{
|
||||
name = "filterAttrsRec";
|
||||
expected = {world = "world";};
|
||||
actual = helpers.filterAttrsRec (n: v: v != null) {
|
||||
hello = null;
|
||||
world = "world";
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "filterJobVariables with store paths";
|
||||
expected = {HELLO = "${pkgs.hello}";};
|
||||
actual = helpers.filterJobVariables true {
|
||||
variables = {
|
||||
HELLO = "${pkgs.hello}";
|
||||
WORLD = "world";
|
||||
};
|
||||
};
|
||||
}
|
||||
{
|
||||
name = "filterJobVariables without store paths";
|
||||
expected = {WORLD = "world";};
|
||||
actual = helpers.filterJobVariables false {
|
||||
variables = {
|
||||
HELLO = "${pkgs.hello}";
|
||||
WORLD = "world";
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
26
tests/pipeline-yamls.nix
Normal file
26
tests/pipeline-yamls.nix
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
self',
|
||||
...
|
||||
}: let
|
||||
cilib = import ./../lib {inherit lib pkgs;};
|
||||
in {
|
||||
nixtest.suites."Pipeline YAMLs" = {
|
||||
pos = __curPos;
|
||||
tests = let
|
||||
jsonFile = file: builtins.fromJSON (builtins.readFile file);
|
||||
in [
|
||||
{
|
||||
name = "default";
|
||||
type = "snapshot";
|
||||
actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:default";
|
||||
}
|
||||
{
|
||||
name = "non-default";
|
||||
type = "snapshot";
|
||||
actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:non-default";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
36
tests/utils.nix
Normal file
36
tests/utils.nix
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
{
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
cilib = import ./../lib {inherit lib pkgs;};
|
||||
in {
|
||||
nixtest.suites."Utils" = {
|
||||
pos = __curPos;
|
||||
tests = [
|
||||
{
|
||||
name = "commitAndPushFiles";
|
||||
type = "script";
|
||||
script = let
|
||||
inherit (cilib) utils;
|
||||
job = builtins.toFile "test" (
|
||||
builtins.unsafeDiscardStringContext (
|
||||
builtins.toJSON (
|
||||
utils.commitAndPushFiles {
|
||||
message = "hello world";
|
||||
files = ["a.md" "b.txt"];
|
||||
} {}
|
||||
)
|
||||
)
|
||||
);
|
||||
in
|
||||
# sh
|
||||
''
|
||||
export PATH=${lib.makeBinPath [pkgs.gnugrep]}
|
||||
grep -q 'git commit -m \\"hello world\\"' ${job}
|
||||
grep -q 'git add a.md b.txt' ${job}
|
||||
'';
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue