mirror of
https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git
synced 2026-02-02 19:35:08 +01:00
chore: split everything up into their own files & add a bunch of tests
This commit is contained in:
parent
b309fb59db
commit
2f197d2c50
20 changed files with 704 additions and 310 deletions
|
|
@ -9,22 +9,12 @@
|
|||
pkgs,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList;
|
||||
cilib = import ./. {inherit lib pkgs;};
|
||||
inherit (cilib.pipeline) mkPipeline;
|
||||
inherit (lib) types mkOption;
|
||||
|
||||
cfg = config.ci.config;
|
||||
|
||||
stdenvMinimal = pkgs.stdenvNoCC.override {
|
||||
cc = null;
|
||||
preHook = "";
|
||||
allowedRequisites = null;
|
||||
initialPath = [pkgs.coreutils pkgs.findutils];
|
||||
extraNativeBuildInputs = [];
|
||||
};
|
||||
|
||||
filterAttrsRec = pred: v:
|
||||
if isAttrs v
|
||||
then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v)
|
||||
else v;
|
||||
|
||||
subType = options: types.submodule {inherit options;};
|
||||
mkNullOption = type:
|
||||
mkOption {
|
||||
|
|
@ -168,228 +158,15 @@
|
|||
};
|
||||
};
|
||||
|
||||
config.legacyPackages = let
|
||||
# NOTE: json is also valid yaml and this removes dependency on jq
|
||||
# and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml
|
||||
# respectively)
|
||||
toYaml = name: value: builtins.toFile name (builtins.toJSON value);
|
||||
mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
|
||||
prepend = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
${key} =
|
||||
arr
|
||||
++ (job.${key} or []);
|
||||
};
|
||||
append = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
${key} = (job.${key} or []) ++ arr;
|
||||
};
|
||||
prependToBeforeScript = prepend "before_script";
|
||||
appendToAfterScript = append "after_script";
|
||||
|
||||
# filter job's variables to either only those containing store paths
|
||||
# or those that do not
|
||||
filterJobVariables = nix: job:
|
||||
lib.concatMapAttrs (
|
||||
name: value:
|
||||
lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) {
|
||||
${name} = value;
|
||||
}
|
||||
)
|
||||
(job.variables or {});
|
||||
in
|
||||
lib.fold (pipeline: acc: acc // pipeline) {} (map (
|
||||
pipeline_name: let
|
||||
pipeline = config.pipelines."${pipeline_name}";
|
||||
jobs = filterAttrsRec (n: v: v != null) pipeline.jobs;
|
||||
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]);
|
||||
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
|
||||
# pro: we don't have to download everything, just the deps for the current job
|
||||
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
|
||||
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
|
||||
jobsMappedForDeps =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithStorePaths = filterJobVariables true job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}";
|
||||
value = stdenvMinimal.mkDerivation {
|
||||
name = "gitlab-ci-job-deps-${key}";
|
||||
dontUnpack = true;
|
||||
installPhase = let
|
||||
script = ''
|
||||
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
|
||||
# variables containing nix derivations:
|
||||
${variableExports}
|
||||
'';
|
||||
in
|
||||
# sh
|
||||
''
|
||||
echo '${script}' > $out
|
||||
chmod +x $out
|
||||
'';
|
||||
};
|
||||
})
|
||||
jobs;
|
||||
# allows the user to directly run the script
|
||||
jobsMappedForScript =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithoutStorePaths = filterJobVariables false job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}";
|
||||
value = let
|
||||
actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" ''
|
||||
# set up deps and environment variables containing store paths
|
||||
. ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"}
|
||||
# normal environment variables
|
||||
${variableExports}
|
||||
# run before_script, script and after_script
|
||||
echo -e "\e[32mRunning before_script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines (job.before_script or [])}
|
||||
{ set +x; } 2>/dev/null
|
||||
echo -e "\e[32mRunning script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines job.script}
|
||||
{ set +x; } 2>/dev/null
|
||||
echo -e "\e[32mRunning after_script...\e[0m"
|
||||
set -x
|
||||
${lib.concatLines (job.after_script or [])}
|
||||
{ set +x; } 2>/dev/null
|
||||
'';
|
||||
sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" ''
|
||||
echo -e "\e[32mSetting up...\e[0m"
|
||||
|
||||
actualJobScript=$1
|
||||
shift
|
||||
|
||||
INCLUDE_DIRTY=false
|
||||
NO_SANDBOX=false
|
||||
KEEP_TMP=false
|
||||
KEEP_ENV=""
|
||||
# parse flags
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--include-dirty)
|
||||
INCLUDE_DIRTY=true
|
||||
shift
|
||||
;;
|
||||
--no-sandbox)
|
||||
NO_SANDBOX=true
|
||||
shift
|
||||
;;
|
||||
--keep-tmp)
|
||||
KEEP_TMP=true
|
||||
shift
|
||||
;;
|
||||
--keep-env)
|
||||
KEEP_ENV="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$NO_SANDBOX" = false ]; then
|
||||
echo "Running with simple sandboxing"
|
||||
if [ "$KEEP_TMP" = false ]; then
|
||||
trap "rm -rf '$TMPDIR'" EXIT
|
||||
else
|
||||
echo "Temp dir will be preserved at: $TMPDIR"
|
||||
fi
|
||||
|
||||
# check if dirty
|
||||
DIRTY_PATCH=""
|
||||
if ! git diff --quiet && ! git diff --staged --quiet; then
|
||||
echo "Warning: working tree is dirty."
|
||||
DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch")
|
||||
git diff --staged > "$DIRTY_PATCH"
|
||||
trap "rm -f '$DIRTY_PATCH'" EXIT
|
||||
fi
|
||||
TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX")
|
||||
git clone . $TMPDIR
|
||||
pushd $TMPDIR >/dev/null
|
||||
if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then
|
||||
echo "Copying dirty changes..."
|
||||
git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes"
|
||||
fi
|
||||
|
||||
echo "Running job in $TMPDIR"
|
||||
env -i $(
|
||||
if [[ -n "$KEEP_ENV" ]]; then
|
||||
IFS=',' read -ra VARS <<< "$KEEP_ENV"
|
||||
for var in "''${VARS[@]}"; do
|
||||
printf '%s=%q ' "$var" "''${!var}"
|
||||
done
|
||||
fi
|
||||
) bash $actualJobScript
|
||||
popd >/dev/null
|
||||
else
|
||||
exec $actualJobScript
|
||||
fi
|
||||
'';
|
||||
in
|
||||
# this way the sandbox helper just needs to be built once
|
||||
pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
|
||||
exec ${lib.getExe sandboxHelper} ${actualJobScript} $@
|
||||
'';
|
||||
})
|
||||
jobs;
|
||||
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
|
||||
jobsPatched =
|
||||
mapAttrs (key: job: {
|
||||
name = key;
|
||||
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
|
||||
builtins.removeAttrs (
|
||||
(prependToBeforeScript [
|
||||
"source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\""
|
||||
]
|
||||
(appendToAfterScript [
|
||||
"finalize_nix_ci"
|
||||
]
|
||||
job))
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
image = job.image;
|
||||
variables =
|
||||
(filterJobVariables false job)
|
||||
// lib.optionalAttrs job.nix.enable-runner-cache {
|
||||
NIX_CI_CACHE_STRATEGY = "runner";
|
||||
};
|
||||
cache =
|
||||
(
|
||||
let
|
||||
c = job.cache or [];
|
||||
in
|
||||
toList c
|
||||
)
|
||||
++ (lib.optional (job.nix.enable-runner-cache) {
|
||||
key = job.nix.runner-cache-key;
|
||||
paths = [".nix-cache/"];
|
||||
});
|
||||
}
|
||||
) ["nix"];
|
||||
})
|
||||
jobs;
|
||||
in
|
||||
# gitlab-ci:pipeline:<name>
|
||||
# gitlab-ci:pipeline:<name>:job:<name>
|
||||
# gitlab-ci:pipeline:<name>:job-deps:<name>
|
||||
{
|
||||
"gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched);
|
||||
}
|
||||
// jobsMappedForDeps
|
||||
// jobsMappedForScript
|
||||
) (builtins.attrNames config.pipelines));
|
||||
config.legacyPackages = lib.fold (pipeline: acc: acc // pipeline) {} (
|
||||
map (
|
||||
pipeline_name:
|
||||
(mkPipeline {
|
||||
pipeline = config.pipelines."${pipeline_name}";
|
||||
name = pipeline_name;
|
||||
}).packages
|
||||
) (builtins.attrNames config.pipelines)
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue