{ flake-parts-lib, lib, ... }: { options.perSystem = flake-parts-lib.mkPerSystemOption ( { config, pkgs, ... }: let inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList; cfg = config.ci.config; stdenvMinimal = pkgs.stdenvNoCC.override { cc = null; preHook = ""; allowedRequisites = null; initialPath = [pkgs.coreutils pkgs.findutils]; extraNativeBuildInputs = []; }; filterAttrsRec = pred: v: if isAttrs v then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) else v; subType = options: types.submodule {inherit options;}; mkNullOption = type: mkOption { default = null; type = types.nullOr type; }; configType = subType { nix-jobs-by-default = mkOption { type = types.bool; default = true; description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; }; }; jobType = subType { # nix ci opts nix = mkOption { type = subType { enable = mkOption { type = types.bool; default = cfg.nix-jobs-by-default; description = "Handle this job as a nix job"; }; deps = mkOption { type = types.listOf types.package; default = []; description = "Dependencies/packages to install for this job"; }; enable-runner-cache = mkOption { type = types.bool; default = false; description = '' Cache this job using the GitLab Runner cache. Warning: useful for tiny jobs, but most of the time it just takes an eternity. ''; }; runner-cache-key = mkOption { type = types.str; default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true"; }; }; default = {}; description = "Configure Nix Gitlab CI for each job individually"; }; # gitlab opts script = mkOption { type = types.listOf types.str; default = []; }; stage = mkOption { type = types.str; default = "test"; }; image = mkOption { type = types.str; default = "$NIX_CI_IMAGE"; }; after_script = mkNullOption (types.listOf types.str); allow_failure = mkNullOption (types.either types.attrs types.bool); artifacts = mkNullOption (types.attrs); before_script = mkNullOption (types.listOf types.str); cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); coverage = mkNullOption (types.str); dependencies = mkNullOption (types.listOf types.str); environment = mkNullOption (types.either types.attrs types.str); extends = mkNullOption (types.str); hooks = mkNullOption (types.attrs); id_tokens = mkNullOption (types.attrs); "inherit" = mkNullOption (types.attrs); interruptible = mkNullOption (types.bool); needs = mkNullOption (types.listOf (types.either types.str types.attrs)); publish = mkNullOption (types.str); pages = mkNullOption (types.attrs); parallel = mkNullOption (types.either types.int types.attrs); release = mkNullOption (types.attrs); retry = mkNullOption (types.either types.int types.attrs); rules = mkNullOption (types.listOf types.attrs); resource_group = mkNullOption (types.str); secrets = mkNullOption (types.attrs); services = mkNullOption (types.listOf types.attrs); start_in = mkNullOption (types.str); tags = mkNullOption (types.listOf types.str); timeout = mkNullOption (types.str); variables = mkNullOption (types.attrs); when = mkNullOption (types.str); }; ciType = subType { config = mkOption { type = configType; description = '' Configuration options for the nix part itself ''; default = {}; }; image = mkNullOption (types.str); variables = mkNullOption (types.attrs); default = mkNullOption (types.attrs); stages = mkNullOption (types.listOf types.str); include = mkNullOption (types.attrs); workflow = mkNullOption (types.attrs); jobs = mkOption { type = types.lazyAttrsOf jobType; default = {}; }; }; in { options = { pipelines = mkOption { type = types.lazyAttrsOf ciType; description = '' Create multiple GitLab CI pipelines. See README.md for more information about how a pipeline is selected. ''; default = {}; apply = op: let # NOTE: show warning if "default" is set and config.ci is not {} legacyMode = config.ci != {}; defaultExists = builtins.hasAttr "default" op; value = { "default" = config.ci; } // op; in if defaultExists && legacyMode then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value else value; }; ci = mkOption { type = ciType; description = '' Note: this is a shorthand for writing `pipelines."default"` Generate a Gitlab CI configuration which can be used to trigger a child pipeline. This will inject code which pre-downloads the nix deps before each job and adds them to PATH. ''; default = {}; }; }; config.legacyPackages = let # NOTE: json is also valid yaml and this removes dependency on jq # and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml # respectively) toYaml = name: value: builtins.toFile name (builtins.toJSON value); mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); prepend = key: arr: job: job // lib.optionalAttrs job.nix.enable { ${key} = arr ++ (job.${key} or []); }; append = key: arr: job: job // lib.optionalAttrs job.nix.enable { ${key} = (job.${key} or []) ++ arr; }; prependToBeforeScript = prepend "before_script"; appendToAfterScript = append "after_script"; # filter job's variables to either only those containing store paths # or those that do not filterJobVariables = nix: job: lib.concatMapAttrs ( name: value: lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { ${name} = value; } ) (job.variables or {}); in lib.fold (pipeline: acc: acc // pipeline) {} (map ( pipeline_name: let pipeline = config.pipelines."${pipeline_name}"; jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); # this allows us to nix build this to get all the mentioned dependencies from the binary cache # pro: we don't have to download everything, just the deps for the current job # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file # we can support different architectures between runners (eg. the arch of the initial runner does not matter) jobsMappedForDeps = mapAttrs (key: job: let variablesWithStorePaths = filterJobVariables true job; variableExports = lib.concatLines ( lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths ); in { name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"; value = stdenvMinimal.mkDerivation { name = "gitlab-ci-job-deps-${key}"; dontUnpack = true; installPhase = let script = '' export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; # variables containing nix derivations: ${variableExports} ''; in # sh '' echo '${script}' > $out chmod +x $out ''; }; }) jobs; # allows the user to directly run the script jobsMappedForScript = mapAttrs (key: job: let variablesWithoutStorePaths = filterJobVariables false job; variableExports = lib.concatLines ( lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths ); in { name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}"; value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' # set up deps and environment variables containing store paths . ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"} # normal environment variables ${variableExports} # run before_script, script and after_script echo -e "\e[32mRunning before_script...\e[0m" ${lib.concatLines (job.before_script or [])} echo -e "\e[32mRunning script...\e[0m" ${lib.concatLines job.script} echo -e "\e[32mRunning after_script...\e[0m" ${lib.concatLines (job.after_script or [])} ''; }) jobs; # build the deps specific for this job before anything, this way the deps should be fetched from the cache jobsPatched = mapAttrs (key: job: { name = key; value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; builtins.removeAttrs ( (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" ] (appendToAfterScript [ "finalize_nix_ci" ] job)) // lib.optionalAttrs job.nix.enable { image = job.image; variables = (filterJobVariables false job) // lib.optionalAttrs job.nix.enable-runner-cache { NIX_CI_CACHE_STRATEGY = "runner"; }; cache = ( let c = job.cache or []; in toList c ) ++ (lib.optional (job.nix.enable-runner-cache) { key = job.nix.runner-cache-key; paths = [".nix-cache/"]; }); } ) ["nix"]; }) jobs; in # gitlab-ci:pipeline: # gitlab-ci:pipeline::job: # gitlab-ci:pipeline::job-deps: { "gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched); } // jobsMappedForDeps // jobsMappedForScript ) (builtins.attrNames config.pipelines)); } ); }