mirror of
https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git
synced 2025-12-12 02:00:13 +01:00
feat(v2): initial v2 implementation
add multi-arch (arm & x64) image add multiple pipelines (ci now creates the "default" pipeline as a shorthand) simplify devenv flake input merge all cache options together, now $NIX_CI_CACHE_STRATEGY decides how the cache works setup_nix_ci and finalize_nix_ci are now flake packages and work standalone the specific image is not needed anymore, any image with the right dependencies works runner cache is not the default anymore (because it sucked most of the time) the pipeline is selected by $NIX_CI_PIPELINE_NAME or if empty by $CI_PIPELINE_SOURCE, so for the old behaviour $NIX_CI_PIPELINE_NAME=default is needed, future work will be needed to handle this more nicely
This commit is contained in:
parent
016e6c9dc7
commit
586fb88b9d
6 changed files with 409 additions and 412 deletions
|
|
@ -9,137 +9,150 @@
|
|||
pkgs,
|
||||
...
|
||||
}: let
|
||||
inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList;
|
||||
cfg = config.ci.config;
|
||||
|
||||
filterAttrsRec = pred: v:
|
||||
if lib.isAttrs v
|
||||
then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v)
|
||||
if isAttrs v
|
||||
then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v)
|
||||
else v;
|
||||
|
||||
subType = options: lib.types.submodule {inherit options;};
|
||||
subType = options: types.submodule {inherit options;};
|
||||
mkNullOption = type:
|
||||
lib.mkOption {
|
||||
mkOption {
|
||||
default = null;
|
||||
type = lib.types.nullOr type;
|
||||
type = types.nullOr type;
|
||||
};
|
||||
|
||||
configType = with lib;
|
||||
subType {
|
||||
default-nix-image = mkOption {
|
||||
type = types.str;
|
||||
default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest";
|
||||
description = "The image to use on nix jobs";
|
||||
};
|
||||
nix-jobs-per-default = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
|
||||
};
|
||||
disable-cache = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE";
|
||||
};
|
||||
cache-key = mkOption {
|
||||
type = types.str;
|
||||
default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
|
||||
description = "Cache key to use for the nix cache";
|
||||
};
|
||||
configType = subType {
|
||||
nix-jobs-per-default = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
|
||||
};
|
||||
jobType = with lib;
|
||||
subType {
|
||||
# nix ci opts
|
||||
nix = mkOption {
|
||||
type = subType {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = cfg.nix-jobs-per-default;
|
||||
description = "Handle this job as a nix job";
|
||||
};
|
||||
deps = mkOption {
|
||||
type = types.listOf types.package;
|
||||
default = [];
|
||||
description = "Dependencies/packages to install for this job";
|
||||
};
|
||||
disable-cache = mkOption {
|
||||
type = types.bool;
|
||||
default = cfg.disable-cache;
|
||||
description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE";
|
||||
};
|
||||
cache-key = mkOption {
|
||||
type = types.str;
|
||||
default = cfg.cache-key;
|
||||
description = "Cache key to use for the nix cache";
|
||||
};
|
||||
};
|
||||
default = {};
|
||||
description = "Configure Nix Gitlab CI for each job individually";
|
||||
};
|
||||
# gitlab opts
|
||||
script = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
stage = mkOption {
|
||||
type = types.str;
|
||||
default = "test";
|
||||
};
|
||||
image = mkOption {
|
||||
type = types.str;
|
||||
default = cfg.default-nix-image;
|
||||
};
|
||||
after_script = mkNullOption (types.listOf types.str);
|
||||
allow_failure = mkNullOption (types.either types.attrs types.bool);
|
||||
artifacts = mkNullOption (types.attrs);
|
||||
before_script = mkNullOption (types.listOf types.str);
|
||||
cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
|
||||
coverage = mkNullOption (types.str);
|
||||
dependencies = mkNullOption (types.listOf types.str);
|
||||
environment = mkNullOption (types.either types.attrs types.str);
|
||||
extends = mkNullOption (types.str);
|
||||
hooks = mkNullOption (types.attrs);
|
||||
id_tokens = mkNullOption (types.attrs);
|
||||
"inherit" = mkNullOption (types.attrs);
|
||||
interruptible = mkNullOption (types.bool);
|
||||
needs = mkNullOption (types.listOf (types.either types.str types.attrs));
|
||||
publish = mkNullOption (types.str);
|
||||
pages = mkNullOption (types.attrs);
|
||||
parallel = mkNullOption (types.either types.int types.attrs);
|
||||
release = mkNullOption (types.attrs);
|
||||
retry = mkNullOption (types.either types.int types.attrs);
|
||||
rules = mkNullOption (types.listOf types.attrs);
|
||||
resource_group = mkNullOption (types.str);
|
||||
secrets = mkNullOption (types.attrs);
|
||||
services = mkNullOption (types.listOf types.attrs);
|
||||
start_in = mkNullOption (types.str);
|
||||
tags = mkNullOption (types.listOf types.str);
|
||||
timeout = mkNullOption (types.str);
|
||||
variables = mkNullOption (types.attrs);
|
||||
when = mkNullOption (types.str);
|
||||
};
|
||||
in {
|
||||
options = with lib; {
|
||||
ci = mkOption {
|
||||
};
|
||||
jobType = subType {
|
||||
# nix ci opts
|
||||
nix = mkOption {
|
||||
type = subType {
|
||||
config = mkOption {
|
||||
type = configType;
|
||||
description = ''
|
||||
Configuration options for the nix part itself
|
||||
'';
|
||||
default = {};
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = cfg.nix-jobs-per-default;
|
||||
description = "Handle this job as a nix job";
|
||||
};
|
||||
image = mkNullOption (types.str);
|
||||
variables = mkNullOption (types.attrs);
|
||||
default = mkNullOption (types.attrs);
|
||||
stages = mkNullOption (types.listOf types.str);
|
||||
include = mkNullOption (types.attrs);
|
||||
workflow = mkNullOption (types.attrs);
|
||||
jobs = mkOption {
|
||||
type = types.lazyAttrsOf jobType;
|
||||
default = {};
|
||||
deps = mkOption {
|
||||
type = types.listOf types.package;
|
||||
default = [];
|
||||
description = "Dependencies/packages to install for this job";
|
||||
};
|
||||
enable-runner-cache = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Cache this job using the GitLab Runner cache.
|
||||
Warning: useful for tiny jobs, but most of the time it just takes an eternity.
|
||||
'';
|
||||
};
|
||||
runner-cache-key = mkOption {
|
||||
type = types.str;
|
||||
default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
|
||||
description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true";
|
||||
};
|
||||
};
|
||||
default = {};
|
||||
description = "Configure Nix Gitlab CI for each job individually";
|
||||
};
|
||||
# gitlab opts
|
||||
script = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
stage = mkOption {
|
||||
type = types.str;
|
||||
default = "test";
|
||||
};
|
||||
image = mkOption {
|
||||
type = types.str;
|
||||
default = "$_NIX_CI_IMAGE";
|
||||
};
|
||||
after_script = mkNullOption (types.listOf types.str);
|
||||
allow_failure = mkNullOption (types.either types.attrs types.bool);
|
||||
artifacts = mkNullOption (types.attrs);
|
||||
before_script = mkNullOption (types.listOf types.str);
|
||||
cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
|
||||
coverage = mkNullOption (types.str);
|
||||
dependencies = mkNullOption (types.listOf types.str);
|
||||
environment = mkNullOption (types.either types.attrs types.str);
|
||||
extends = mkNullOption (types.str);
|
||||
hooks = mkNullOption (types.attrs);
|
||||
id_tokens = mkNullOption (types.attrs);
|
||||
"inherit" = mkNullOption (types.attrs);
|
||||
interruptible = mkNullOption (types.bool);
|
||||
needs = mkNullOption (types.listOf (types.either types.str types.attrs));
|
||||
publish = mkNullOption (types.str);
|
||||
pages = mkNullOption (types.attrs);
|
||||
parallel = mkNullOption (types.either types.int types.attrs);
|
||||
release = mkNullOption (types.attrs);
|
||||
retry = mkNullOption (types.either types.int types.attrs);
|
||||
rules = mkNullOption (types.listOf types.attrs);
|
||||
resource_group = mkNullOption (types.str);
|
||||
secrets = mkNullOption (types.attrs);
|
||||
services = mkNullOption (types.listOf types.attrs);
|
||||
start_in = mkNullOption (types.str);
|
||||
tags = mkNullOption (types.listOf types.str);
|
||||
timeout = mkNullOption (types.str);
|
||||
variables = mkNullOption (types.attrs);
|
||||
when = mkNullOption (types.str);
|
||||
};
|
||||
|
||||
ciType = subType {
|
||||
config = mkOption {
|
||||
type = configType;
|
||||
description = ''
|
||||
Configuration options for the nix part itself
|
||||
'';
|
||||
default = {};
|
||||
};
|
||||
image = mkNullOption (types.str);
|
||||
variables = mkNullOption (types.attrs);
|
||||
default = mkNullOption (types.attrs);
|
||||
stages = mkNullOption (types.listOf types.str);
|
||||
include = mkNullOption (types.attrs);
|
||||
workflow = mkNullOption (types.attrs);
|
||||
jobs = mkOption {
|
||||
type = types.lazyAttrsOf jobType;
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
in {
|
||||
options = {
|
||||
pipelines = mkOption {
|
||||
type = types.lazyAttrsOf ciType;
|
||||
description = ''
|
||||
Create multiple GitLab CI pipelines.
|
||||
|
||||
See README.md for more information about how a pipeline is selected.
|
||||
'';
|
||||
default = {};
|
||||
apply = op: let
|
||||
# NOTE: show warning if "default" is set and config.ci is not {}
|
||||
legacyMode = config.ci != {};
|
||||
defaultExists = builtins.hasAttr "default" op;
|
||||
value =
|
||||
{
|
||||
"default" = config.ci;
|
||||
}
|
||||
// op;
|
||||
in
|
||||
if defaultExists && legacyMode
|
||||
then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value
|
||||
else value;
|
||||
};
|
||||
ci = mkOption {
|
||||
type = ciType;
|
||||
description = ''
|
||||
Note: this is a shorthand for writing `pipelines."default"`
|
||||
|
||||
Generate a Gitlab CI configuration which can be used to trigger a child pipeline.
|
||||
This will inject code which pre-downloads the nix deps before each job and adds them to PATH.
|
||||
'';
|
||||
|
|
@ -175,94 +188,100 @@
|
|||
}
|
||||
)
|
||||
(job.variables or {});
|
||||
|
||||
jobs = filterAttrsRec (n: v: v != null) config.ci.jobs;
|
||||
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]);
|
||||
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
|
||||
# pro: we don't have to download everything, just the deps for the current job
|
||||
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
|
||||
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
|
||||
jobsMappedForDeps =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithStorePaths = filterJobVariables true job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci-job-deps:${key}";
|
||||
value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" ''
|
||||
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
|
||||
${variableExports}
|
||||
'';
|
||||
})
|
||||
jobs;
|
||||
# allows the user to directly run the script
|
||||
jobsMappedForScript =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithStorePaths = filterJobVariables false job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci-job:${key}";
|
||||
value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
|
||||
# set up deps and environment variables containing store paths
|
||||
. ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"}
|
||||
# normal environment variables
|
||||
${variableExports}
|
||||
# run before_script, script and after_script
|
||||
echo -e "\e[32mRunning before_script...\e[0m"
|
||||
${lib.concatLines (job.before_script or [])}
|
||||
echo -e "\e[32mRunning script...\e[0m"
|
||||
${lib.concatLines job.script}
|
||||
echo -e "\e[32mRunning after_script...\e[0m"
|
||||
${lib.concatLines (job.after_script or [])}
|
||||
'';
|
||||
})
|
||||
jobs;
|
||||
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
|
||||
jobsPatched =
|
||||
mapAttrs (key: job: {
|
||||
name = key;
|
||||
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
|
||||
builtins.removeAttrs (
|
||||
(prependToBeforeScript [
|
||||
"source setup_nix_ci ${key}"
|
||||
]
|
||||
(appendToAfterScript [
|
||||
"finalize_nix_ci"
|
||||
]
|
||||
job))
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
image = job.image;
|
||||
variables =
|
||||
(filterJobVariables false job)
|
||||
// lib.optionalAttrs job.nix.disable-cache {
|
||||
NIX_CI_DISABLE_CACHE = "yes";
|
||||
};
|
||||
cache =
|
||||
(
|
||||
let
|
||||
c = job.cache or [];
|
||||
in
|
||||
if builtins.isList c
|
||||
then c
|
||||
else [c]
|
||||
)
|
||||
++ (lib.optional (!job.nix.disable-cache) {
|
||||
key = job.nix.cache-key;
|
||||
paths = [".nix-cache/"];
|
||||
});
|
||||
}
|
||||
) ["nix"];
|
||||
})
|
||||
jobs;
|
||||
in
|
||||
{
|
||||
gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched);
|
||||
}
|
||||
// jobsMappedForDeps
|
||||
// jobsMappedForScript;
|
||||
lib.fold (pipeline: acc: acc // pipeline) {} (map (
|
||||
pipeline_name: let
|
||||
pipeline = config.pipelines."${pipeline_name}";
|
||||
jobs = filterAttrsRec (n: v: v != null) pipeline.jobs;
|
||||
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]);
|
||||
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
|
||||
# pro: we don't have to download everything, just the deps for the current job
|
||||
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
|
||||
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
|
||||
jobsMappedForDeps =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithStorePaths = filterJobVariables true job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}";
|
||||
value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" ''
|
||||
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
|
||||
# variables containing nix derivations:
|
||||
${variableExports}
|
||||
'';
|
||||
})
|
||||
jobs;
|
||||
# allows the user to directly run the script
|
||||
jobsMappedForScript =
|
||||
mapAttrs (key: job: let
|
||||
variablesWithoutStorePaths = filterJobVariables false job;
|
||||
variableExports = lib.concatLines (
|
||||
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
|
||||
);
|
||||
in {
|
||||
name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}";
|
||||
value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
|
||||
# set up deps and environment variables containing store paths
|
||||
. ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"}
|
||||
# normal environment variables
|
||||
${variableExports}
|
||||
# run before_script, script and after_script
|
||||
echo -e "\e[32mRunning before_script...\e[0m"
|
||||
${lib.concatLines (job.before_script or [])}
|
||||
echo -e "\e[32mRunning script...\e[0m"
|
||||
${lib.concatLines job.script}
|
||||
echo -e "\e[32mRunning after_script...\e[0m"
|
||||
${lib.concatLines (job.after_script or [])}
|
||||
'';
|
||||
})
|
||||
jobs;
|
||||
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
|
||||
jobsPatched =
|
||||
mapAttrs (key: job: {
|
||||
name = key;
|
||||
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
|
||||
builtins.removeAttrs (
|
||||
(prependToBeforeScript [
|
||||
"source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\""
|
||||
]
|
||||
(appendToAfterScript [
|
||||
"finalize_nix_ci"
|
||||
]
|
||||
job))
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
image = job.image;
|
||||
variables =
|
||||
(filterJobVariables false job)
|
||||
// lib.optionalAttrs job.nix.enable-runner-cache {
|
||||
_NIX_CI_CACHE_STRATEGY = "runner";
|
||||
};
|
||||
cache =
|
||||
(
|
||||
let
|
||||
c = job.cache or [];
|
||||
in
|
||||
toList c
|
||||
)
|
||||
++ (lib.optional (job.nix.enable-runner-cache) {
|
||||
key = job.nix.runner-cache-key;
|
||||
paths = [".nix-cache/"];
|
||||
});
|
||||
}
|
||||
) ["nix"];
|
||||
})
|
||||
jobs;
|
||||
in
|
||||
# gitlab-ci:pipeline:<name>
|
||||
# gitlab-ci:pipeline:<name>:job:<name>
|
||||
# gitlab-ci:pipeline:<name>:job-deps:<name>
|
||||
{
|
||||
"gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched);
|
||||
}
|
||||
// jobsMappedForDeps
|
||||
// jobsMappedForScript
|
||||
) (builtins.attrNames config.pipelines));
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue