mirror of
https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git
synced 2025-12-12 10:10:06 +01:00
technofab
586fb88b9d
add multi-arch (arm & x64) image add multiple pipelines (ci now creates the "default" pipeline as a shorthand) simplify devenv flake input merge all cache options together, now $NIX_CI_CACHE_STRATEGY decides how the cache works setup_nix_ci and finalize_nix_ci are now flake packages and work standalone the specific image is not needed anymore, any image with the right dependencies works runner cache is not the default anymore (because it sucked most of the time) the pipeline is selected by $NIX_CI_PIPELINE_NAME or if empty by $CI_PIPELINE_SOURCE, so for the old behaviour $NIX_CI_PIPELINE_NAME=default is needed, future work will be needed to handle this more nicely
287 lines
11 KiB
Nix
287 lines
11 KiB
Nix
{
|
|
flake-parts-lib,
|
|
lib,
|
|
...
|
|
}: {
|
|
options.perSystem = flake-parts-lib.mkPerSystemOption (
|
|
{
|
|
config,
|
|
pkgs,
|
|
...
|
|
}: let
|
|
inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList;
|
|
cfg = config.ci.config;
|
|
|
|
filterAttrsRec = pred: v:
|
|
if isAttrs v
|
|
then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v)
|
|
else v;
|
|
|
|
subType = options: types.submodule {inherit options;};
|
|
mkNullOption = type:
|
|
mkOption {
|
|
default = null;
|
|
type = types.nullOr type;
|
|
};
|
|
|
|
configType = subType {
|
|
nix-jobs-per-default = mkOption {
|
|
type = types.bool;
|
|
default = true;
|
|
description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
|
|
};
|
|
};
|
|
jobType = subType {
|
|
# nix ci opts
|
|
nix = mkOption {
|
|
type = subType {
|
|
enable = mkOption {
|
|
type = types.bool;
|
|
default = cfg.nix-jobs-per-default;
|
|
description = "Handle this job as a nix job";
|
|
};
|
|
deps = mkOption {
|
|
type = types.listOf types.package;
|
|
default = [];
|
|
description = "Dependencies/packages to install for this job";
|
|
};
|
|
enable-runner-cache = mkOption {
|
|
type = types.bool;
|
|
default = false;
|
|
description = ''
|
|
Cache this job using the GitLab Runner cache.
|
|
Warning: useful for tiny jobs, but most of the time it just takes an eternity.
|
|
'';
|
|
};
|
|
runner-cache-key = mkOption {
|
|
type = types.str;
|
|
default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
|
|
description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true";
|
|
};
|
|
};
|
|
default = {};
|
|
description = "Configure Nix Gitlab CI for each job individually";
|
|
};
|
|
# gitlab opts
|
|
script = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [];
|
|
};
|
|
stage = mkOption {
|
|
type = types.str;
|
|
default = "test";
|
|
};
|
|
image = mkOption {
|
|
type = types.str;
|
|
default = "$_NIX_CI_IMAGE";
|
|
};
|
|
after_script = mkNullOption (types.listOf types.str);
|
|
allow_failure = mkNullOption (types.either types.attrs types.bool);
|
|
artifacts = mkNullOption (types.attrs);
|
|
before_script = mkNullOption (types.listOf types.str);
|
|
cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
|
|
coverage = mkNullOption (types.str);
|
|
dependencies = mkNullOption (types.listOf types.str);
|
|
environment = mkNullOption (types.either types.attrs types.str);
|
|
extends = mkNullOption (types.str);
|
|
hooks = mkNullOption (types.attrs);
|
|
id_tokens = mkNullOption (types.attrs);
|
|
"inherit" = mkNullOption (types.attrs);
|
|
interruptible = mkNullOption (types.bool);
|
|
needs = mkNullOption (types.listOf (types.either types.str types.attrs));
|
|
publish = mkNullOption (types.str);
|
|
pages = mkNullOption (types.attrs);
|
|
parallel = mkNullOption (types.either types.int types.attrs);
|
|
release = mkNullOption (types.attrs);
|
|
retry = mkNullOption (types.either types.int types.attrs);
|
|
rules = mkNullOption (types.listOf types.attrs);
|
|
resource_group = mkNullOption (types.str);
|
|
secrets = mkNullOption (types.attrs);
|
|
services = mkNullOption (types.listOf types.attrs);
|
|
start_in = mkNullOption (types.str);
|
|
tags = mkNullOption (types.listOf types.str);
|
|
timeout = mkNullOption (types.str);
|
|
variables = mkNullOption (types.attrs);
|
|
when = mkNullOption (types.str);
|
|
};
|
|
|
|
ciType = subType {
|
|
config = mkOption {
|
|
type = configType;
|
|
description = ''
|
|
Configuration options for the nix part itself
|
|
'';
|
|
default = {};
|
|
};
|
|
image = mkNullOption (types.str);
|
|
variables = mkNullOption (types.attrs);
|
|
default = mkNullOption (types.attrs);
|
|
stages = mkNullOption (types.listOf types.str);
|
|
include = mkNullOption (types.attrs);
|
|
workflow = mkNullOption (types.attrs);
|
|
jobs = mkOption {
|
|
type = types.lazyAttrsOf jobType;
|
|
default = {};
|
|
};
|
|
};
|
|
in {
|
|
options = {
|
|
pipelines = mkOption {
|
|
type = types.lazyAttrsOf ciType;
|
|
description = ''
|
|
Create multiple GitLab CI pipelines.
|
|
|
|
See README.md for more information about how a pipeline is selected.
|
|
'';
|
|
default = {};
|
|
apply = op: let
|
|
# NOTE: show warning if "default" is set and config.ci is not {}
|
|
legacyMode = config.ci != {};
|
|
defaultExists = builtins.hasAttr "default" op;
|
|
value =
|
|
{
|
|
"default" = config.ci;
|
|
}
|
|
// op;
|
|
in
|
|
if defaultExists && legacyMode
|
|
then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value
|
|
else value;
|
|
};
|
|
ci = mkOption {
|
|
type = ciType;
|
|
description = ''
|
|
Note: this is a shorthand for writing `pipelines."default"`
|
|
|
|
Generate a Gitlab CI configuration which can be used to trigger a child pipeline.
|
|
This will inject code which pre-downloads the nix deps before each job and adds them to PATH.
|
|
'';
|
|
default = {};
|
|
};
|
|
};
|
|
|
|
config.legacyPackages = let
|
|
toYaml = (pkgs.formats.yaml {}).generate;
|
|
mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
|
|
prepend = key: arr: job:
|
|
job
|
|
// lib.optionalAttrs job.nix.enable {
|
|
${key} =
|
|
arr
|
|
++ (job.${key} or []);
|
|
};
|
|
append = key: arr: job:
|
|
job
|
|
// lib.optionalAttrs job.nix.enable {
|
|
${key} = (job.${key} or []) ++ arr;
|
|
};
|
|
prependToBeforeScript = prepend "before_script";
|
|
appendToAfterScript = append "after_script";
|
|
|
|
# filter job's variables to either only those containing store paths
|
|
# or those that do not
|
|
filterJobVariables = nix: job:
|
|
lib.concatMapAttrs (
|
|
name: value:
|
|
lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) {
|
|
${name} = value;
|
|
}
|
|
)
|
|
(job.variables or {});
|
|
in
|
|
lib.fold (pipeline: acc: acc // pipeline) {} (map (
|
|
pipeline_name: let
|
|
pipeline = config.pipelines."${pipeline_name}";
|
|
jobs = filterAttrsRec (n: v: v != null) pipeline.jobs;
|
|
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]);
|
|
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
|
|
# pro: we don't have to download everything, just the deps for the current job
|
|
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
|
|
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
|
|
jobsMappedForDeps =
|
|
mapAttrs (key: job: let
|
|
variablesWithStorePaths = filterJobVariables true job;
|
|
variableExports = lib.concatLines (
|
|
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
|
);
|
|
in {
|
|
name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}";
|
|
value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" ''
|
|
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
|
|
# variables containing nix derivations:
|
|
${variableExports}
|
|
'';
|
|
})
|
|
jobs;
|
|
# allows the user to directly run the script
|
|
jobsMappedForScript =
|
|
mapAttrs (key: job: let
|
|
variablesWithoutStorePaths = filterJobVariables false job;
|
|
variableExports = lib.concatLines (
|
|
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
|
|
);
|
|
in {
|
|
name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}";
|
|
value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
|
|
# set up deps and environment variables containing store paths
|
|
. ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"}
|
|
# normal environment variables
|
|
${variableExports}
|
|
# run before_script, script and after_script
|
|
echo -e "\e[32mRunning before_script...\e[0m"
|
|
${lib.concatLines (job.before_script or [])}
|
|
echo -e "\e[32mRunning script...\e[0m"
|
|
${lib.concatLines job.script}
|
|
echo -e "\e[32mRunning after_script...\e[0m"
|
|
${lib.concatLines (job.after_script or [])}
|
|
'';
|
|
})
|
|
jobs;
|
|
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
|
|
jobsPatched =
|
|
mapAttrs (key: job: {
|
|
name = key;
|
|
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
|
|
builtins.removeAttrs (
|
|
(prependToBeforeScript [
|
|
"source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\""
|
|
]
|
|
(appendToAfterScript [
|
|
"finalize_nix_ci"
|
|
]
|
|
job))
|
|
// lib.optionalAttrs job.nix.enable {
|
|
image = job.image;
|
|
variables =
|
|
(filterJobVariables false job)
|
|
// lib.optionalAttrs job.nix.enable-runner-cache {
|
|
_NIX_CI_CACHE_STRATEGY = "runner";
|
|
};
|
|
cache =
|
|
(
|
|
let
|
|
c = job.cache or [];
|
|
in
|
|
toList c
|
|
)
|
|
++ (lib.optional (job.nix.enable-runner-cache) {
|
|
key = job.nix.runner-cache-key;
|
|
paths = [".nix-cache/"];
|
|
});
|
|
}
|
|
) ["nix"];
|
|
})
|
|
jobs;
|
|
in
|
|
# gitlab-ci:pipeline:<name>
|
|
# gitlab-ci:pipeline:<name>:job:<name>
|
|
# gitlab-ci:pipeline:<name>:job-deps:<name>
|
|
{
|
|
"gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched);
|
|
}
|
|
// jobsMappedForDeps
|
|
// jobsMappedForScript
|
|
) (builtins.attrNames config.pipelines));
|
|
}
|
|
);
|
|
}
|