From 586fb88b9d51a662431a4281a2f66274b1d2c919 Mon Sep 17 00:00:00 2001 From: technofab Date: Fri, 21 Feb 2025 12:24:54 +0100 Subject: [PATCH] feat(v2): initial v2 implementation add multi-arch (arm & x64) image add multiple pipelines (ci now creates the "default" pipeline as a shorthand) simplify devenv flake input merge all cache options together, now $NIX_CI_CACHE_STRATEGY decides how the cache works setup_nix_ci and finalize_nix_ci are now flake packages and work standalone the specific image is not needed anymore, any image with the right dependencies works runner cache is not the default anymore (because it sucked most of the time) the pipeline is selected by $NIX_CI_PIPELINE_NAME or if empty by $CI_PIPELINE_SOURCE, so for the old behaviour $NIX_CI_PIPELINE_NAME=default is needed, future work will be needed to handle this more nicely --- .gitlab-ci.yml | 34 ++- README.md | 16 +- flake.lock | 76 ++----- flake.nix | 240 ++++++++++---------- lib/flakeModule.nix | 425 +++++++++++++++++++----------------- templates/nix-gitlab-ci.yml | 30 ++- 6 files changed, 409 insertions(+), 412 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0393fb..84edac4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,26 +10,40 @@ build:image: stage: build-images parallel: matrix: - - VARIANT: ["", "-cachix", "-attic"] + - ARCH: ["x86_64-linux", "aarch64-linux"] image: nixpkgs/nix-flakes:latest before_script: - - nix profile install nixpkgs#skopeo + - nix profile install nixpkgs#buildah - export PATH="$PATH:$HOME/.nix-profile/bin" script: - - nix build .#image${VARIANT} + - nix build .#image --system $ARCH + after_script: + - install -D result dist/nix-ci-$ARCH.tar.gz + artifacts: + paths: + - dist +deploy:image: + stage: build-images + needs: + - build:image + before_script: + - export REGISTRY_AUTH_FILE=''${HOME}/auth.json + - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + - mkdir -p /etc/containers && echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json + - mkdir -p /var/tmp + script: - export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-} - - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_SHORT_SHA}${VARIANT}" + - buildah manifest create localhost/nix-ci + - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz + - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz + - buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_SHORT_SHA} # branches - | if [ -z "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${NORMALIZED_BRANCH/main/latest}${VARIANT}"; + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${NORMALIZED_BRANCH/main/latest} fi # tags - | if [ -n "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_TAG}${VARIANT}"; + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_TAG} fi diff --git a/README.md b/README.md index 9735feb..da2b36b 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Also makes it possible to split CI parts in a separate module which can be impor ... perSystem = {pkgs, ...}: { + # ci is a shortcut and creates a "default" pipeline ci = { stages = ["test"]; jobs = { @@ -33,6 +34,11 @@ Also makes it possible to split CI parts in a separate module which can be impor }; }; }; + # runs on a merge request for example + pipelines."merge_request_event" = { + stages = ["some_stage"]; + jobs = { ... }; + }; ... } } @@ -43,9 +49,6 @@ Also makes it possible to split CI parts in a separate module which can be impor # .gitlab-ci.yml include: - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: use the latest version (try not to use latest) - inputs: - # specify inputs here, for example: - image_tag: latest-cachix ``` ## Utilities @@ -63,16 +66,15 @@ The `build:nix-ci` job has a different special environment variable `NIX_CI_FORC You can run any job's script (+ before and after) locally with Nix for easier testing: ```sh -nix run .#gitlab-ci-job: +# / pipeline name, like "default" +nix run .#gitlab-ci:pipeline::job: ``` -There is also `.#gitlab-ci-job-deps:` which generates and exports the required environment variables for each job: +There is also `.#gitlab-ci:pipeline::job-deps:` which generates and exports the required environment variables for each job: - PATH (with all deps) - any custom env variables which contain store paths to not break stuff when switching archs -Please see #8 for some issues and further improvements on this. - ## Thanks to Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix diff --git a/flake.lock b/flake.lock index dd864f4..ff34ba8 100644 --- a/flake.lock +++ b/flake.lock @@ -32,9 +32,7 @@ "inputs": { "cachix": "cachix", "flake-compat": "flake-compat", - "git-hooks": [ - "git-hooks" - ], + "git-hooks": "git-hooks", "nix": "nix", "nixpkgs": "nixpkgs_3" }, @@ -68,22 +66,6 @@ "type": "github" } }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -126,17 +108,21 @@ }, "git-hooks": { "inputs": { - "flake-compat": "flake-compat_2", + "flake-compat": [ + "devenv" + ], "gitignore": "gitignore", - "nixpkgs": "nixpkgs_4", - "nixpkgs-stable": "nixpkgs-stable" + "nixpkgs": [ + "devenv", + "nixpkgs" + ] }, "locked": { - "lastModified": 1732021966, - "narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", + "lastModified": 1737465171, + "narHash": "sha256-R10v2hoJRLq8jcL4syVFag7nIGE7m13qO48wRIukWNg=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "3308484d1a443fc5bc92012435d79e80458fe43c", + "rev": "9364dc02281ce2d37a1f55b6e51f7c0f65a75f17", "type": "github" }, "original": { @@ -148,6 +134,7 @@ "gitignore": { "inputs": { "nixpkgs": [ + "devenv", "git-hooks", "nixpkgs" ] @@ -243,22 +230,6 @@ "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" } }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1730741070, - "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", - "type": "github" - } - }, "nixpkgs_2": { "locked": { "lastModified": 1717432640, @@ -292,22 +263,6 @@ } }, "nixpkgs_4": { - "locked": { - "lastModified": 1730768919, - "narHash": "sha256-8AKquNnnSaJRXZxc5YmF/WfmxiHX6MMZZasRP6RRQkE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a04d33c0c3f1a59a2c1cb0c6e34cd24500e5a1dc", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_5": { "locked": { "lastModified": 1732238832, "narHash": "sha256-sQxuJm8rHY20xq6Ah+GwIUkF95tWjGRd1X8xF+Pkk38=", @@ -323,7 +278,7 @@ "type": "github" } }, - "nixpkgs_6": { + "nixpkgs_5": { "locked": { "lastModified": 1731890469, "narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=", @@ -343,8 +298,7 @@ "inputs": { "devenv": "devenv", "flake-parts": "flake-parts_2", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_5", + "nixpkgs": "nixpkgs_4", "systems": "systems", "treefmt-nix": "treefmt-nix" } @@ -366,7 +320,7 @@ }, "treefmt-nix": { "inputs": { - "nixpkgs": "nixpkgs_6" + "nixpkgs": "nixpkgs_5" }, "locked": { "lastModified": 1732643199, diff --git a/flake.nix b/flake.nix index 596b693..f07964e 100644 --- a/flake.nix +++ b/flake.nix @@ -39,16 +39,15 @@ }; }; }; + # should set the "default" pipeline ci = { - # use the image built in the parent pipeline for dogfooding - config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA"; stages = ["test"]; jobs = { "test" = { stage = "test"; nix = { deps = [pkgs.hello pkgs.curl]; - disable-cache = false; + enable-runner-cache = true; }; variables = { TEST = "test"; @@ -60,6 +59,11 @@ "echo $TEST $TEST_WITH_DERIVATION" ]; }; + "test-default" = { + stage = "test"; + nix.deps = [pkgs.hello]; + script = ["hello"]; + }; "test-non-nix" = { nix.enable = false; stage = "test"; @@ -70,130 +74,130 @@ }; }; }; + pipelines."non-default" = { + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + script = [ + "echo Hello from another pipeline" + ]; + }; + }; + }; packages = let - setupScript = extra_setup: - pkgs.writeShellScriptBin "setup_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" - nix path-info --all > /tmp/nix-store-before + setupScript = pkgs.writeShellScriptBin "setup_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:nix_setup\\r\\e[0KSetting up Nix CI" + nix path-info --all > /tmp/nix-store-before - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${extra_setup} - else - echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration" - fi - - export NIX_CONFIG=" - extra-trusted-public-keys = $NIX_PUBLIC_KEYS - extra-trusted-substituters = $NIX_SUBSTITUTERS - extra-substituters = $NIX_SUBSTITUTERS - $NIX_CONFIG - $NIX_EXTRA_CONFIG - " - echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + if [ -z "$_NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($_NIX_CI_CACHE_STRATEGY)" + case "$_NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + echo "Runner Cache: $RUNNER_CACHE" + export NIX_CONFIG="$NIX_CONFIG + extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true + extra-substituters = $RUNNER_CACHE?priority=10&trusted=true + " + ;; + "attic") + echo "Attic Cache: $ATTIC_CACHE" + attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true + attic use "$ATTIC_CACHE" || true + ;; + "cachix") + echo "Cachix Cache: $CACHIX_CACHE" + cachix use "$CACHIX_CACHE" || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$_NIX_CI_CACHE_STRATEGY'" + ;; + esac + echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K" + else + echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..." + fi # load the job's deps only if the name was passed if [[ ! -z $1 ]]; then - echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job" - nix build .#gitlab-ci-job-deps:$1 + echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job" + nix build .#$1 source $(readlink -f result) echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" fi - ''; - finalizeScript = push_command: - pkgs.writeShellScriptBin "finalize_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache" + echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + ''; + finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci\\r\\e[0KFinalizing Nix CI..." nix path-info --all > /tmp/nix-store-after - ${pkgs.diffutils}/bin/diff --new-line-format="%L" \ + echo "Finding new paths..." + NEW_PATHS=$(${pkgs.diffutils}/bin/diff --new-line-format="%L" \ --old-line-format="" --unchanged-line-format="" \ - /tmp/nix-store-before /tmp/nix-store-after \ - | { - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${push_command} - else - ${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; } - fi - } - echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" - ''; - mkImage = extraPackages: - pkgs.dockerTools.buildImage { - name = "nix-gitlab-ci"; - fromImage = pkgs.dockerTools.pullImage { - imageName = "nixpkgs/nix-flakes"; - imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; - sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; - finalImageName = "nixpkgs/nix-flakes"; - finalImageTag = "latest"; - }; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = - [ - pkgs.gitMinimal - pkgs.gnugrep - ] - ++ extraPackages; - pathsToLink = ["/bin"]; - }; - }; + /tmp/nix-store-before /tmp/nix-store-after) + COUNT=$(${pkgs.busybox}/bin/wc -l <<<"$NEW_PATHS") + if [ -z "$_NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($_NIX_CI_CACHE_STRATEGY)" + echo $NEW_PATHS | { + case "$_NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + # add ^* to all store paths ending in .drv (prevent warning log spam) + ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true + ;; + "attic") + attic push --stdin ci:$ATTIC_CACHE || true + ;; + "cachix") + cachix push $CACHIX_CACHE || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$_NIX_CI_CACHE_STRATEGY'" + ;; + esac + } + echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" + else + echo "Caching disabled, not uploading $COUNT new store entries..." + fi + echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K" + ''; in { - setup-script = - setupScript - # sh - '' - # extra_setup - true - ''; - finalize-script = - finalizeScript - # sh - '' - # push_command - true - ''; - image = mkImage [ - (setupScript - # sh - '' - cachedir="$(pwd)/.nix-cache" - echo "Configuring caching with the Runner Cache in $cachedir..." - export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true" - '') - (finalizeScript - # sh - '' - # add ^* to all store paths ending in .drv (prevent warning log spam) - ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true - '') - ]; - image-cachix = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with cachix..." - ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true - '') - (finalizeScript - # sh - '' - ${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true - '') - ]; - image-attic = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with attic..." - ${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true - ${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true - '') - (finalizeScript - # sh - '' - ${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true - '') - ]; + setup-script = setupScript; + finalize-script = finalizeScript; + image = pkgs.dockerTools.buildImage { + name = "nix-ci"; + fromImage = pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-flakes"; + # nix run nixpkgs#nix-prefetch-docker -- --image-name nixpkgs/nix-flakes --image-tag latest --arch --os linux + imageDigest = "sha256:95bce4317c15dfab3babac5a6d19d3ed41e31a02a8aaf3d4f6639778cb763b0a"; + sha256 = + if pkgs.stdenv.hostPlatform.isAarch64 + then "DMlSaP+ZVqxd9NxdFydGyfkuJdmOW5jt5iM/7cDyTEM=" + else "mfTNlGOpThanLlLQ2lL1RTcHqZJWdqUafYDZMeZPWEk="; + finalImageName = "nixpkgs/nix-flakes"; + finalImageTag = "latest"; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [ + pkgs.gitMinimal + pkgs.gnugrep + pkgs.cachix + pkgs.attic-client + setupScript + finalizeScript + ]; + pathsToLink = ["/bin"]; + }; + }; }; checks = packages; @@ -206,11 +210,7 @@ # flake & devenv related flake-parts.url = "github:hercules-ci/flake-parts"; systems.url = "github:nix-systems/default-linux"; - devenv = { - url = "github:cachix/devenv"; - inputs.git-hooks.follows = "git-hooks"; - }; - git-hooks.url = "github:cachix/git-hooks.nix"; + devenv.url = "github:cachix/devenv"; treefmt-nix.url = "github:numtide/treefmt-nix"; }; diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 64f7926..5b9b86a 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,137 +9,150 @@ pkgs, ... }: let + inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList; cfg = config.ci.config; filterAttrsRec = pred: v: - if lib.isAttrs v - then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v) + if isAttrs v + then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) else v; - subType = options: lib.types.submodule {inherit options;}; + subType = options: types.submodule {inherit options;}; mkNullOption = type: - lib.mkOption { + mkOption { default = null; - type = lib.types.nullOr type; + type = types.nullOr type; }; - configType = with lib; - subType { - default-nix-image = mkOption { - type = types.str; - default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest"; - description = "The image to use on nix jobs"; - }; - nix-jobs-per-default = mkOption { - type = types.bool; - default = true; - description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; - }; - disable-cache = mkOption { - type = types.bool; - default = false; - description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; - description = "Cache key to use for the nix cache"; - }; + configType = subType { + nix-jobs-per-default = mkOption { + type = types.bool; + default = true; + description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; }; - jobType = with lib; - subType { - # nix ci opts - nix = mkOption { - type = subType { - enable = mkOption { - type = types.bool; - default = cfg.nix-jobs-per-default; - description = "Handle this job as a nix job"; - }; - deps = mkOption { - type = types.listOf types.package; - default = []; - description = "Dependencies/packages to install for this job"; - }; - disable-cache = mkOption { - type = types.bool; - default = cfg.disable-cache; - description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = cfg.cache-key; - description = "Cache key to use for the nix cache"; - }; - }; - default = {}; - description = "Configure Nix Gitlab CI for each job individually"; - }; - # gitlab opts - script = mkOption { - type = types.listOf types.str; - default = []; - }; - stage = mkOption { - type = types.str; - default = "test"; - }; - image = mkOption { - type = types.str; - default = cfg.default-nix-image; - }; - after_script = mkNullOption (types.listOf types.str); - allow_failure = mkNullOption (types.either types.attrs types.bool); - artifacts = mkNullOption (types.attrs); - before_script = mkNullOption (types.listOf types.str); - cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); - coverage = mkNullOption (types.str); - dependencies = mkNullOption (types.listOf types.str); - environment = mkNullOption (types.either types.attrs types.str); - extends = mkNullOption (types.str); - hooks = mkNullOption (types.attrs); - id_tokens = mkNullOption (types.attrs); - "inherit" = mkNullOption (types.attrs); - interruptible = mkNullOption (types.bool); - needs = mkNullOption (types.listOf (types.either types.str types.attrs)); - publish = mkNullOption (types.str); - pages = mkNullOption (types.attrs); - parallel = mkNullOption (types.either types.int types.attrs); - release = mkNullOption (types.attrs); - retry = mkNullOption (types.either types.int types.attrs); - rules = mkNullOption (types.listOf types.attrs); - resource_group = mkNullOption (types.str); - secrets = mkNullOption (types.attrs); - services = mkNullOption (types.listOf types.attrs); - start_in = mkNullOption (types.str); - tags = mkNullOption (types.listOf types.str); - timeout = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - when = mkNullOption (types.str); - }; - in { - options = with lib; { - ci = mkOption { + }; + jobType = subType { + # nix ci opts + nix = mkOption { type = subType { - config = mkOption { - type = configType; - description = '' - Configuration options for the nix part itself - ''; - default = {}; + enable = mkOption { + type = types.bool; + default = cfg.nix-jobs-per-default; + description = "Handle this job as a nix job"; }; - image = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - default = mkNullOption (types.attrs); - stages = mkNullOption (types.listOf types.str); - include = mkNullOption (types.attrs); - workflow = mkNullOption (types.attrs); - jobs = mkOption { - type = types.lazyAttrsOf jobType; - default = {}; + deps = mkOption { + type = types.listOf types.package; + default = []; + description = "Dependencies/packages to install for this job"; + }; + enable-runner-cache = mkOption { + type = types.bool; + default = false; + description = '' + Cache this job using the GitLab Runner cache. + Warning: useful for tiny jobs, but most of the time it just takes an eternity. + ''; + }; + runner-cache-key = mkOption { + type = types.str; + default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; + description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true"; }; }; + default = {}; + description = "Configure Nix Gitlab CI for each job individually"; + }; + # gitlab opts + script = mkOption { + type = types.listOf types.str; + default = []; + }; + stage = mkOption { + type = types.str; + default = "test"; + }; + image = mkOption { + type = types.str; + default = "$_NIX_CI_IMAGE"; + }; + after_script = mkNullOption (types.listOf types.str); + allow_failure = mkNullOption (types.either types.attrs types.bool); + artifacts = mkNullOption (types.attrs); + before_script = mkNullOption (types.listOf types.str); + cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); + coverage = mkNullOption (types.str); + dependencies = mkNullOption (types.listOf types.str); + environment = mkNullOption (types.either types.attrs types.str); + extends = mkNullOption (types.str); + hooks = mkNullOption (types.attrs); + id_tokens = mkNullOption (types.attrs); + "inherit" = mkNullOption (types.attrs); + interruptible = mkNullOption (types.bool); + needs = mkNullOption (types.listOf (types.either types.str types.attrs)); + publish = mkNullOption (types.str); + pages = mkNullOption (types.attrs); + parallel = mkNullOption (types.either types.int types.attrs); + release = mkNullOption (types.attrs); + retry = mkNullOption (types.either types.int types.attrs); + rules = mkNullOption (types.listOf types.attrs); + resource_group = mkNullOption (types.str); + secrets = mkNullOption (types.attrs); + services = mkNullOption (types.listOf types.attrs); + start_in = mkNullOption (types.str); + tags = mkNullOption (types.listOf types.str); + timeout = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + when = mkNullOption (types.str); + }; + + ciType = subType { + config = mkOption { + type = configType; description = '' + Configuration options for the nix part itself + ''; + default = {}; + }; + image = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + default = mkNullOption (types.attrs); + stages = mkNullOption (types.listOf types.str); + include = mkNullOption (types.attrs); + workflow = mkNullOption (types.attrs); + jobs = mkOption { + type = types.lazyAttrsOf jobType; + default = {}; + }; + }; + in { + options = { + pipelines = mkOption { + type = types.lazyAttrsOf ciType; + description = '' + Create multiple GitLab CI pipelines. + + See README.md for more information about how a pipeline is selected. + ''; + default = {}; + apply = op: let + # NOTE: show warning if "default" is set and config.ci is not {} + legacyMode = config.ci != {}; + defaultExists = builtins.hasAttr "default" op; + value = + { + "default" = config.ci; + } + // op; + in + if defaultExists && legacyMode + then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value + else value; + }; + ci = mkOption { + type = ciType; + description = '' + Note: this is a shorthand for writing `pipelines."default"` + Generate a Gitlab CI configuration which can be used to trigger a child pipeline. This will inject code which pre-downloads the nix deps before each job and adds them to PATH. ''; @@ -175,94 +188,100 @@ } ) (job.variables or {}); - - jobs = filterAttrsRec (n: v: v != null) config.ci.jobs; - rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job-deps:${key}"; - value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' - export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; - ${variableExports} - ''; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables false job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job:${key}"; - value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - # set up deps and environment variables containing store paths - . ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - ${lib.concatLines (job.before_script or [])} - echo -e "\e[32mRunning script...\e[0m" - ${lib.concatLines job.script} - echo -e "\e[32mRunning after_script...\e[0m" - ${lib.concatLines (job.after_script or [])} - ''; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - mapAttrs (key: job: { - name = key; - value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - builtins.removeAttrs ( - (prependToBeforeScript [ - "source setup_nix_ci ${key}" - ] - (appendToAfterScript [ - "finalize_nix_ci" - ] - job)) - // lib.optionalAttrs job.nix.enable { - image = job.image; - variables = - (filterJobVariables false job) - // lib.optionalAttrs job.nix.disable-cache { - NIX_CI_DISABLE_CACHE = "yes"; - }; - cache = - ( - let - c = job.cache or []; - in - if builtins.isList c - then c - else [c] - ) - ++ (lib.optional (!job.nix.disable-cache) { - key = job.nix.cache-key; - paths = [".nix-cache/"]; - }); - } - ) ["nix"]; - }) - jobs; in - { - gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript; + lib.fold (pipeline: acc: acc // pipeline) {} (map ( + pipeline_name: let + pipeline = config.pipelines."${pipeline_name}"; + jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; + rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + mapAttrs (key: job: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"; + value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' + export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; + # variables containing nix derivations: + ${variableExports} + ''; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + mapAttrs (key: job: let + variablesWithoutStorePaths = filterJobVariables false job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}"; + value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + # set up deps and environment variables containing store paths + . ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + ${lib.concatLines (job.before_script or [])} + echo -e "\e[32mRunning script...\e[0m" + ${lib.concatLines job.script} + echo -e "\e[32mRunning after_script...\e[0m" + ${lib.concatLines (job.after_script or [])} + ''; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + mapAttrs (key: job: { + name = key; + value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + builtins.removeAttrs ( + (prependToBeforeScript [ + "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" + ] + (appendToAfterScript [ + "finalize_nix_ci" + ] + job)) + // lib.optionalAttrs job.nix.enable { + image = job.image; + variables = + (filterJobVariables false job) + // lib.optionalAttrs job.nix.enable-runner-cache { + _NIX_CI_CACHE_STRATEGY = "runner"; + }; + cache = + ( + let + c = job.cache or []; + in + toList c + ) + ++ (lib.optional (job.nix.enable-runner-cache) { + key = job.nix.runner-cache-key; + paths = [".nix-cache/"]; + }); + } + ) ["nix"]; + }) + jobs; + in + # gitlab-ci:pipeline: + # gitlab-ci:pipeline::job: + # gitlab-ci:pipeline::job-deps: + { + "gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript + ) (builtins.attrNames config.pipelines)); } ); } diff --git a/templates/nix-gitlab-ci.yml b/templates/nix-gitlab-ci.yml index 911556a..dd1e79a 100644 --- a/templates/nix-gitlab-ci.yml +++ b/templates/nix-gitlab-ci.yml @@ -1,29 +1,37 @@ spec: inputs: - image_tag: + cache_strategy: type: string - description: "latest | latest-cachix | latest-attic etc." - default: latest + description: | + (empty for auto) | none | runner | cachix | attic + When left empty $NIX_CI_CACHE_STRATEGY will be used, which defaults to none + default: "" cache_files: type: array description: | Files to use as the cache key for the generated pipeline yaml. If you use "ci.nix" to define CI, add that here for example default: ["flake.nix", "flake.lock"] - disable_cache: - type: string - description: | - Disables any caching provided by this component. Set to any non-empty value to disable caching. - default: "" --- stages: - build - trigger variables: - NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]${NIX_CI_DISABLE_CACHE:-}" + # which version of the image should be used + _NIX_CI_VERSION: ${NIX_CI_VERSION} + _NIX_CI_IMAGE: ${NIX_CI_IMAGE:-registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${_NIX_CI_VERSION}} + # force build the pipeline yaml + _NIX_CI_FORCE_BUILD: ${NIX_CI_FORCE_BUILD} + # disable caching on the child pipeline jobs + _NIX_CI_DISABLE_CACHE: ${NIX_CI_DISABLE_CACHE} + # type of cache strategy to use (none, runner, attic, cachix) + _CACHE_STRATEGY_TMP: $[[ inputs.cache_strategy ]] + _NIX_CI_CACHE_STRATEGY: ${NIX_CI_CACHE_STRATEGY:-${_CACHE_STRATEGY_TMP:-none}} + # for multiple pipelines + _NIX_CI_PIPELINE_NAME: ${NIX_CI_PIPELINE_NAME:-${CI_PIPELINE_SOURCE:-default}} nix-ci:build: stage: build - image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]] + image: $_NIX_CI_IMAGE cache: - key: files: $[[ inputs.cache_files ]] @@ -36,7 +44,7 @@ nix-ci:build: # generated-gitlab-ci.yml exists in the cache - '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true' # allow the user to manually skip the cache (when the key files are not correctly configured etc.) - - '[ -n "$NIX_CI_FORCE_BUILD" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true' + - '[ -n "$_NIX_CI_FORCE_BUILD" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true' # only setup when we need to generate the pipeline yaml - 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi' script: