diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0393fb..84edac4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,26 +10,40 @@ build:image: stage: build-images parallel: matrix: - - VARIANT: ["", "-cachix", "-attic"] + - ARCH: ["x86_64-linux", "aarch64-linux"] image: nixpkgs/nix-flakes:latest before_script: - - nix profile install nixpkgs#skopeo + - nix profile install nixpkgs#buildah - export PATH="$PATH:$HOME/.nix-profile/bin" script: - - nix build .#image${VARIANT} + - nix build .#image --system $ARCH + after_script: + - install -D result dist/nix-ci-$ARCH.tar.gz + artifacts: + paths: + - dist +deploy:image: + stage: build-images + needs: + - build:image + before_script: + - export REGISTRY_AUTH_FILE=''${HOME}/auth.json + - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + - mkdir -p /etc/containers && echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json + - mkdir -p /var/tmp + script: - export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-} - - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_SHORT_SHA}${VARIANT}" + - buildah manifest create localhost/nix-ci + - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz + - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz + - buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_SHORT_SHA} # branches - | if [ -z "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${NORMALIZED_BRANCH/main/latest}${VARIANT}"; + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${NORMALIZED_BRANCH/main/latest} fi # tags - | if [ -n "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_TAG}${VARIANT}"; + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_TAG} fi diff --git a/README.md b/README.md index 9735feb..da2b36b 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Also makes it possible to split CI parts in a separate module which can be impor ... perSystem = {pkgs, ...}: { + # ci is a shortcut and creates a "default" pipeline ci = { stages = ["test"]; jobs = { @@ -33,6 +34,11 @@ Also makes it possible to split CI parts in a separate module which can be impor }; }; }; + # runs on a merge request for example + pipelines."merge_request_event" = { + stages = ["some_stage"]; + jobs = { ... }; + }; ... } } @@ -43,9 +49,6 @@ Also makes it possible to split CI parts in a separate module which can be impor # .gitlab-ci.yml include: - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: use the latest version (try not to use latest) - inputs: - # specify inputs here, for example: - image_tag: latest-cachix ``` ## Utilities @@ -63,16 +66,15 @@ The `build:nix-ci` job has a different special environment variable `NIX_CI_FORC You can run any job's script (+ before and after) locally with Nix for easier testing: ```sh -nix run .#gitlab-ci-job: +# / pipeline name, like "default" +nix run .#gitlab-ci:pipeline::job: ``` -There is also `.#gitlab-ci-job-deps:` which generates and exports the required environment variables for each job: +There is also `.#gitlab-ci:pipeline::job-deps:` which generates and exports the required environment variables for each job: - PATH (with all deps) - any custom env variables which contain store paths to not break stuff when switching archs -Please see #8 for some issues and further improvements on this. - ## Thanks to Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix diff --git a/flake.lock b/flake.lock index dd864f4..ff34ba8 100644 --- a/flake.lock +++ b/flake.lock @@ -32,9 +32,7 @@ "inputs": { "cachix": "cachix", "flake-compat": "flake-compat", - "git-hooks": [ - "git-hooks" - ], + "git-hooks": "git-hooks", "nix": "nix", "nixpkgs": "nixpkgs_3" }, @@ -68,22 +66,6 @@ "type": "github" } }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -126,17 +108,21 @@ }, "git-hooks": { "inputs": { - "flake-compat": "flake-compat_2", + "flake-compat": [ + "devenv" + ], "gitignore": "gitignore", - "nixpkgs": "nixpkgs_4", - "nixpkgs-stable": "nixpkgs-stable" + "nixpkgs": [ + "devenv", + "nixpkgs" + ] }, "locked": { - "lastModified": 1732021966, - "narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", + "lastModified": 1737465171, + "narHash": "sha256-R10v2hoJRLq8jcL4syVFag7nIGE7m13qO48wRIukWNg=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "3308484d1a443fc5bc92012435d79e80458fe43c", + "rev": "9364dc02281ce2d37a1f55b6e51f7c0f65a75f17", "type": "github" }, "original": { @@ -148,6 +134,7 @@ "gitignore": { "inputs": { "nixpkgs": [ + "devenv", "git-hooks", "nixpkgs" ] @@ -243,22 +230,6 @@ "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" } }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1730741070, - "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", - "type": "github" - } - }, "nixpkgs_2": { "locked": { "lastModified": 1717432640, @@ -292,22 +263,6 @@ } }, "nixpkgs_4": { - "locked": { - "lastModified": 1730768919, - "narHash": "sha256-8AKquNnnSaJRXZxc5YmF/WfmxiHX6MMZZasRP6RRQkE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a04d33c0c3f1a59a2c1cb0c6e34cd24500e5a1dc", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_5": { "locked": { "lastModified": 1732238832, "narHash": "sha256-sQxuJm8rHY20xq6Ah+GwIUkF95tWjGRd1X8xF+Pkk38=", @@ -323,7 +278,7 @@ "type": "github" } }, - "nixpkgs_6": { + "nixpkgs_5": { "locked": { "lastModified": 1731890469, "narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=", @@ -343,8 +298,7 @@ "inputs": { "devenv": "devenv", "flake-parts": "flake-parts_2", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_5", + "nixpkgs": "nixpkgs_4", "systems": "systems", "treefmt-nix": "treefmt-nix" } @@ -366,7 +320,7 @@ }, "treefmt-nix": { "inputs": { - "nixpkgs": "nixpkgs_6" + "nixpkgs": "nixpkgs_5" }, "locked": { "lastModified": 1732643199, diff --git a/flake.nix b/flake.nix index 596b693..f07964e 100644 --- a/flake.nix +++ b/flake.nix @@ -39,16 +39,15 @@ }; }; }; + # should set the "default" pipeline ci = { - # use the image built in the parent pipeline for dogfooding - config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA"; stages = ["test"]; jobs = { "test" = { stage = "test"; nix = { deps = [pkgs.hello pkgs.curl]; - disable-cache = false; + enable-runner-cache = true; }; variables = { TEST = "test"; @@ -60,6 +59,11 @@ "echo $TEST $TEST_WITH_DERIVATION" ]; }; + "test-default" = { + stage = "test"; + nix.deps = [pkgs.hello]; + script = ["hello"]; + }; "test-non-nix" = { nix.enable = false; stage = "test"; @@ -70,130 +74,130 @@ }; }; }; + pipelines."non-default" = { + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + script = [ + "echo Hello from another pipeline" + ]; + }; + }; + }; packages = let - setupScript = extra_setup: - pkgs.writeShellScriptBin "setup_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" - nix path-info --all > /tmp/nix-store-before + setupScript = pkgs.writeShellScriptBin "setup_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:nix_setup\\r\\e[0KSetting up Nix CI" + nix path-info --all > /tmp/nix-store-before - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${extra_setup} - else - echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration" - fi - - export NIX_CONFIG=" - extra-trusted-public-keys = $NIX_PUBLIC_KEYS - extra-trusted-substituters = $NIX_SUBSTITUTERS - extra-substituters = $NIX_SUBSTITUTERS - $NIX_CONFIG - $NIX_EXTRA_CONFIG - " - echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + if [ -z "$_NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($_NIX_CI_CACHE_STRATEGY)" + case "$_NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + echo "Runner Cache: $RUNNER_CACHE" + export NIX_CONFIG="$NIX_CONFIG + extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true + extra-substituters = $RUNNER_CACHE?priority=10&trusted=true + " + ;; + "attic") + echo "Attic Cache: $ATTIC_CACHE" + attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true + attic use "$ATTIC_CACHE" || true + ;; + "cachix") + echo "Cachix Cache: $CACHIX_CACHE" + cachix use "$CACHIX_CACHE" || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$_NIX_CI_CACHE_STRATEGY'" + ;; + esac + echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K" + else + echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..." + fi # load the job's deps only if the name was passed if [[ ! -z $1 ]]; then - echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job" - nix build .#gitlab-ci-job-deps:$1 + echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job" + nix build .#$1 source $(readlink -f result) echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" fi - ''; - finalizeScript = push_command: - pkgs.writeShellScriptBin "finalize_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache" + echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + ''; + finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci\\r\\e[0KFinalizing Nix CI..." nix path-info --all > /tmp/nix-store-after - ${pkgs.diffutils}/bin/diff --new-line-format="%L" \ + echo "Finding new paths..." + NEW_PATHS=$(${pkgs.diffutils}/bin/diff --new-line-format="%L" \ --old-line-format="" --unchanged-line-format="" \ - /tmp/nix-store-before /tmp/nix-store-after \ - | { - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${push_command} - else - ${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; } - fi - } - echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" - ''; - mkImage = extraPackages: - pkgs.dockerTools.buildImage { - name = "nix-gitlab-ci"; - fromImage = pkgs.dockerTools.pullImage { - imageName = "nixpkgs/nix-flakes"; - imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; - sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; - finalImageName = "nixpkgs/nix-flakes"; - finalImageTag = "latest"; - }; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = - [ - pkgs.gitMinimal - pkgs.gnugrep - ] - ++ extraPackages; - pathsToLink = ["/bin"]; - }; - }; + /tmp/nix-store-before /tmp/nix-store-after) + COUNT=$(${pkgs.busybox}/bin/wc -l <<<"$NEW_PATHS") + if [ -z "$_NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($_NIX_CI_CACHE_STRATEGY)" + echo $NEW_PATHS | { + case "$_NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + # add ^* to all store paths ending in .drv (prevent warning log spam) + ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true + ;; + "attic") + attic push --stdin ci:$ATTIC_CACHE || true + ;; + "cachix") + cachix push $CACHIX_CACHE || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$_NIX_CI_CACHE_STRATEGY'" + ;; + esac + } + echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" + else + echo "Caching disabled, not uploading $COUNT new store entries..." + fi + echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K" + ''; in { - setup-script = - setupScript - # sh - '' - # extra_setup - true - ''; - finalize-script = - finalizeScript - # sh - '' - # push_command - true - ''; - image = mkImage [ - (setupScript - # sh - '' - cachedir="$(pwd)/.nix-cache" - echo "Configuring caching with the Runner Cache in $cachedir..." - export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true" - '') - (finalizeScript - # sh - '' - # add ^* to all store paths ending in .drv (prevent warning log spam) - ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true - '') - ]; - image-cachix = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with cachix..." - ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true - '') - (finalizeScript - # sh - '' - ${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true - '') - ]; - image-attic = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with attic..." - ${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true - ${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true - '') - (finalizeScript - # sh - '' - ${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true - '') - ]; + setup-script = setupScript; + finalize-script = finalizeScript; + image = pkgs.dockerTools.buildImage { + name = "nix-ci"; + fromImage = pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-flakes"; + # nix run nixpkgs#nix-prefetch-docker -- --image-name nixpkgs/nix-flakes --image-tag latest --arch --os linux + imageDigest = "sha256:95bce4317c15dfab3babac5a6d19d3ed41e31a02a8aaf3d4f6639778cb763b0a"; + sha256 = + if pkgs.stdenv.hostPlatform.isAarch64 + then "DMlSaP+ZVqxd9NxdFydGyfkuJdmOW5jt5iM/7cDyTEM=" + else "mfTNlGOpThanLlLQ2lL1RTcHqZJWdqUafYDZMeZPWEk="; + finalImageName = "nixpkgs/nix-flakes"; + finalImageTag = "latest"; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [ + pkgs.gitMinimal + pkgs.gnugrep + pkgs.cachix + pkgs.attic-client + setupScript + finalizeScript + ]; + pathsToLink = ["/bin"]; + }; + }; }; checks = packages; @@ -206,11 +210,7 @@ # flake & devenv related flake-parts.url = "github:hercules-ci/flake-parts"; systems.url = "github:nix-systems/default-linux"; - devenv = { - url = "github:cachix/devenv"; - inputs.git-hooks.follows = "git-hooks"; - }; - git-hooks.url = "github:cachix/git-hooks.nix"; + devenv.url = "github:cachix/devenv"; treefmt-nix.url = "github:numtide/treefmt-nix"; }; diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 64f7926..5b9b86a 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,137 +9,150 @@ pkgs, ... }: let + inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList; cfg = config.ci.config; filterAttrsRec = pred: v: - if lib.isAttrs v - then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v) + if isAttrs v + then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) else v; - subType = options: lib.types.submodule {inherit options;}; + subType = options: types.submodule {inherit options;}; mkNullOption = type: - lib.mkOption { + mkOption { default = null; - type = lib.types.nullOr type; + type = types.nullOr type; }; - configType = with lib; - subType { - default-nix-image = mkOption { - type = types.str; - default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest"; - description = "The image to use on nix jobs"; - }; - nix-jobs-per-default = mkOption { - type = types.bool; - default = true; - description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; - }; - disable-cache = mkOption { - type = types.bool; - default = false; - description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; - description = "Cache key to use for the nix cache"; - }; + configType = subType { + nix-jobs-per-default = mkOption { + type = types.bool; + default = true; + description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; }; - jobType = with lib; - subType { - # nix ci opts - nix = mkOption { - type = subType { - enable = mkOption { - type = types.bool; - default = cfg.nix-jobs-per-default; - description = "Handle this job as a nix job"; - }; - deps = mkOption { - type = types.listOf types.package; - default = []; - description = "Dependencies/packages to install for this job"; - }; - disable-cache = mkOption { - type = types.bool; - default = cfg.disable-cache; - description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = cfg.cache-key; - description = "Cache key to use for the nix cache"; - }; - }; - default = {}; - description = "Configure Nix Gitlab CI for each job individually"; - }; - # gitlab opts - script = mkOption { - type = types.listOf types.str; - default = []; - }; - stage = mkOption { - type = types.str; - default = "test"; - }; - image = mkOption { - type = types.str; - default = cfg.default-nix-image; - }; - after_script = mkNullOption (types.listOf types.str); - allow_failure = mkNullOption (types.either types.attrs types.bool); - artifacts = mkNullOption (types.attrs); - before_script = mkNullOption (types.listOf types.str); - cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); - coverage = mkNullOption (types.str); - dependencies = mkNullOption (types.listOf types.str); - environment = mkNullOption (types.either types.attrs types.str); - extends = mkNullOption (types.str); - hooks = mkNullOption (types.attrs); - id_tokens = mkNullOption (types.attrs); - "inherit" = mkNullOption (types.attrs); - interruptible = mkNullOption (types.bool); - needs = mkNullOption (types.listOf (types.either types.str types.attrs)); - publish = mkNullOption (types.str); - pages = mkNullOption (types.attrs); - parallel = mkNullOption (types.either types.int types.attrs); - release = mkNullOption (types.attrs); - retry = mkNullOption (types.either types.int types.attrs); - rules = mkNullOption (types.listOf types.attrs); - resource_group = mkNullOption (types.str); - secrets = mkNullOption (types.attrs); - services = mkNullOption (types.listOf types.attrs); - start_in = mkNullOption (types.str); - tags = mkNullOption (types.listOf types.str); - timeout = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - when = mkNullOption (types.str); - }; - in { - options = with lib; { - ci = mkOption { + }; + jobType = subType { + # nix ci opts + nix = mkOption { type = subType { - config = mkOption { - type = configType; - description = '' - Configuration options for the nix part itself - ''; - default = {}; + enable = mkOption { + type = types.bool; + default = cfg.nix-jobs-per-default; + description = "Handle this job as a nix job"; }; - image = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - default = mkNullOption (types.attrs); - stages = mkNullOption (types.listOf types.str); - include = mkNullOption (types.attrs); - workflow = mkNullOption (types.attrs); - jobs = mkOption { - type = types.lazyAttrsOf jobType; - default = {}; + deps = mkOption { + type = types.listOf types.package; + default = []; + description = "Dependencies/packages to install for this job"; + }; + enable-runner-cache = mkOption { + type = types.bool; + default = false; + description = '' + Cache this job using the GitLab Runner cache. + Warning: useful for tiny jobs, but most of the time it just takes an eternity. + ''; + }; + runner-cache-key = mkOption { + type = types.str; + default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; + description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true"; }; }; + default = {}; + description = "Configure Nix Gitlab CI for each job individually"; + }; + # gitlab opts + script = mkOption { + type = types.listOf types.str; + default = []; + }; + stage = mkOption { + type = types.str; + default = "test"; + }; + image = mkOption { + type = types.str; + default = "$_NIX_CI_IMAGE"; + }; + after_script = mkNullOption (types.listOf types.str); + allow_failure = mkNullOption (types.either types.attrs types.bool); + artifacts = mkNullOption (types.attrs); + before_script = mkNullOption (types.listOf types.str); + cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); + coverage = mkNullOption (types.str); + dependencies = mkNullOption (types.listOf types.str); + environment = mkNullOption (types.either types.attrs types.str); + extends = mkNullOption (types.str); + hooks = mkNullOption (types.attrs); + id_tokens = mkNullOption (types.attrs); + "inherit" = mkNullOption (types.attrs); + interruptible = mkNullOption (types.bool); + needs = mkNullOption (types.listOf (types.either types.str types.attrs)); + publish = mkNullOption (types.str); + pages = mkNullOption (types.attrs); + parallel = mkNullOption (types.either types.int types.attrs); + release = mkNullOption (types.attrs); + retry = mkNullOption (types.either types.int types.attrs); + rules = mkNullOption (types.listOf types.attrs); + resource_group = mkNullOption (types.str); + secrets = mkNullOption (types.attrs); + services = mkNullOption (types.listOf types.attrs); + start_in = mkNullOption (types.str); + tags = mkNullOption (types.listOf types.str); + timeout = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + when = mkNullOption (types.str); + }; + + ciType = subType { + config = mkOption { + type = configType; description = '' + Configuration options for the nix part itself + ''; + default = {}; + }; + image = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + default = mkNullOption (types.attrs); + stages = mkNullOption (types.listOf types.str); + include = mkNullOption (types.attrs); + workflow = mkNullOption (types.attrs); + jobs = mkOption { + type = types.lazyAttrsOf jobType; + default = {}; + }; + }; + in { + options = { + pipelines = mkOption { + type = types.lazyAttrsOf ciType; + description = '' + Create multiple GitLab CI pipelines. + + See README.md for more information about how a pipeline is selected. + ''; + default = {}; + apply = op: let + # NOTE: show warning if "default" is set and config.ci is not {} + legacyMode = config.ci != {}; + defaultExists = builtins.hasAttr "default" op; + value = + { + "default" = config.ci; + } + // op; + in + if defaultExists && legacyMode + then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value + else value; + }; + ci = mkOption { + type = ciType; + description = '' + Note: this is a shorthand for writing `pipelines."default"` + Generate a Gitlab CI configuration which can be used to trigger a child pipeline. This will inject code which pre-downloads the nix deps before each job and adds them to PATH. ''; @@ -175,94 +188,100 @@ } ) (job.variables or {}); - - jobs = filterAttrsRec (n: v: v != null) config.ci.jobs; - rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job-deps:${key}"; - value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' - export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; - ${variableExports} - ''; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables false job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job:${key}"; - value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - # set up deps and environment variables containing store paths - . ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - ${lib.concatLines (job.before_script or [])} - echo -e "\e[32mRunning script...\e[0m" - ${lib.concatLines job.script} - echo -e "\e[32mRunning after_script...\e[0m" - ${lib.concatLines (job.after_script or [])} - ''; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - mapAttrs (key: job: { - name = key; - value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - builtins.removeAttrs ( - (prependToBeforeScript [ - "source setup_nix_ci ${key}" - ] - (appendToAfterScript [ - "finalize_nix_ci" - ] - job)) - // lib.optionalAttrs job.nix.enable { - image = job.image; - variables = - (filterJobVariables false job) - // lib.optionalAttrs job.nix.disable-cache { - NIX_CI_DISABLE_CACHE = "yes"; - }; - cache = - ( - let - c = job.cache or []; - in - if builtins.isList c - then c - else [c] - ) - ++ (lib.optional (!job.nix.disable-cache) { - key = job.nix.cache-key; - paths = [".nix-cache/"]; - }); - } - ) ["nix"]; - }) - jobs; in - { - gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript; + lib.fold (pipeline: acc: acc // pipeline) {} (map ( + pipeline_name: let + pipeline = config.pipelines."${pipeline_name}"; + jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; + rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + mapAttrs (key: job: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"; + value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' + export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; + # variables containing nix derivations: + ${variableExports} + ''; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + mapAttrs (key: job: let + variablesWithoutStorePaths = filterJobVariables false job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}"; + value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + # set up deps and environment variables containing store paths + . ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + ${lib.concatLines (job.before_script or [])} + echo -e "\e[32mRunning script...\e[0m" + ${lib.concatLines job.script} + echo -e "\e[32mRunning after_script...\e[0m" + ${lib.concatLines (job.after_script or [])} + ''; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + mapAttrs (key: job: { + name = key; + value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + builtins.removeAttrs ( + (prependToBeforeScript [ + "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" + ] + (appendToAfterScript [ + "finalize_nix_ci" + ] + job)) + // lib.optionalAttrs job.nix.enable { + image = job.image; + variables = + (filterJobVariables false job) + // lib.optionalAttrs job.nix.enable-runner-cache { + _NIX_CI_CACHE_STRATEGY = "runner"; + }; + cache = + ( + let + c = job.cache or []; + in + toList c + ) + ++ (lib.optional (job.nix.enable-runner-cache) { + key = job.nix.runner-cache-key; + paths = [".nix-cache/"]; + }); + } + ) ["nix"]; + }) + jobs; + in + # gitlab-ci:pipeline: + # gitlab-ci:pipeline::job: + # gitlab-ci:pipeline::job-deps: + { + "gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript + ) (builtins.attrNames config.pipelines)); } ); } diff --git a/templates/nix-gitlab-ci.yml b/templates/nix-gitlab-ci.yml index 911556a..dd1e79a 100644 --- a/templates/nix-gitlab-ci.yml +++ b/templates/nix-gitlab-ci.yml @@ -1,29 +1,37 @@ spec: inputs: - image_tag: + cache_strategy: type: string - description: "latest | latest-cachix | latest-attic etc." - default: latest + description: | + (empty for auto) | none | runner | cachix | attic + When left empty $NIX_CI_CACHE_STRATEGY will be used, which defaults to none + default: "" cache_files: type: array description: | Files to use as the cache key for the generated pipeline yaml. If you use "ci.nix" to define CI, add that here for example default: ["flake.nix", "flake.lock"] - disable_cache: - type: string - description: | - Disables any caching provided by this component. Set to any non-empty value to disable caching. - default: "" --- stages: - build - trigger variables: - NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]${NIX_CI_DISABLE_CACHE:-}" + # which version of the image should be used + _NIX_CI_VERSION: ${NIX_CI_VERSION} + _NIX_CI_IMAGE: ${NIX_CI_IMAGE:-registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${_NIX_CI_VERSION}} + # force build the pipeline yaml + _NIX_CI_FORCE_BUILD: ${NIX_CI_FORCE_BUILD} + # disable caching on the child pipeline jobs + _NIX_CI_DISABLE_CACHE: ${NIX_CI_DISABLE_CACHE} + # type of cache strategy to use (none, runner, attic, cachix) + _CACHE_STRATEGY_TMP: $[[ inputs.cache_strategy ]] + _NIX_CI_CACHE_STRATEGY: ${NIX_CI_CACHE_STRATEGY:-${_CACHE_STRATEGY_TMP:-none}} + # for multiple pipelines + _NIX_CI_PIPELINE_NAME: ${NIX_CI_PIPELINE_NAME:-${CI_PIPELINE_SOURCE:-default}} nix-ci:build: stage: build - image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]] + image: $_NIX_CI_IMAGE cache: - key: files: $[[ inputs.cache_files ]] @@ -36,7 +44,7 @@ nix-ci:build: # generated-gitlab-ci.yml exists in the cache - '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true' # allow the user to manually skip the cache (when the key files are not correctly configured etc.) - - '[ -n "$NIX_CI_FORCE_BUILD" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true' + - '[ -n "$_NIX_CI_FORCE_BUILD" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true' # only setup when we need to generate the pipeline yaml - 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi' script: