From 3a05d7a12829bedda7a864c87fe11f35365fce2a Mon Sep 17 00:00:00 2001 From: TECHNOFAB Date: Sun, 17 Mar 2024 15:44:22 +0000 Subject: [PATCH] feat: caching support --- .gitlab-ci.yml | 14 ++++--- flake.lock | 14 +++---- flake.nix | 99 +++++++++++++++++++++++++++++++++++++------------ flakeModule.nix | 17 +++++---- gitlab-ci.yml | 9 ++--- 5 files changed, 103 insertions(+), 50 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c641d7..281be3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,16 +7,18 @@ stages: build:image: stage: build + parallel: + matrix: + - VARIANT: [ "", "-cachix", "-attic" ] image: nixpkgs/nix-flakes:latest before_script: - nix profile install nixpkgs#skopeo - export PATH="$PATH:$HOME/.nix-profile/bin" - - mkdir -p /var/tmp script: - - nix build .#image - - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" + - nix build .#image${VARIANT} + - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" - "docker://$CI_REGISTRY_IMAGE/nix-ci:latest" - - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" + "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_BRANCH/main/latest}${VARIANT}" + - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" - "docker://$CI_REGISTRY_IMAGE/nix-ci:$(date +"%m-%d-%y")" + "docker://$CI_REGISTRY_IMAGE/nix-ci:$(date +"%m-%d-%y")${VARIANT}" diff --git a/flake.lock b/flake.lock index 1db7b50..0468da7 100644 --- a/flake.lock +++ b/flake.lock @@ -9,17 +9,17 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1704159259, - "narHash": "sha256-OOEFibN9JQBepVeqxSNSBr6JUmdoZiH263ogms2jk0k=", - "owner": "zhaofengli", + "lastModified": 1710680783, + "narHash": "sha256-exrsUUk/VSCG2Y3Sr/dkST8vwICzab3fe8Je81t/SfM=", + "owner": "TECHNOFAB", "repo": "attic", - "rev": "e6bedf1869f382cfc51b69848d6e09d51585ead6", - "type": "github" + "rev": "b6d8d5ca53ef15910042a39e81f35383370c1112", + "type": "gitlab" }, "original": { - "owner": "zhaofengli", + "owner": "TECHNOFAB", "repo": "attic", - "type": "github" + "type": "gitlab" } }, "crane": { diff --git a/flake.nix b/flake.nix index f28f790..bc5d37c 100644 --- a/flake.nix +++ b/flake.nix @@ -24,16 +24,15 @@ system, inputs', ... - }: { + }: rec { formatter = pkgs.alejandra; - devenv.shells = { - default = { - packages = [pkgs.dive pkgs.skopeo]; + devenv.shells.default = { + containers = pkgs.lib.mkForce {}; + packages = with pkgs; [dive skopeo]; - pre-commit = { - hooks = { - alejandra.enable = true; - }; + pre-commit = { + hooks = { + alejandra.enable = true; }; }; }; @@ -68,23 +67,75 @@ }; }; - packages = { - image = pkgs.dockerTools.buildImage { - name = "nix-gitlab-ci"; - fromImage = pkgs.dockerTools.pullImage { - imageName = "nixpkgs/nix-flakes"; - imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; - sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; - finalImageName = "nixpkgs/nix-flakes"; - finalImageTag = "latest"; + packages = let + setupScript = extra_setup: + pkgs.writeShellScriptBin "setup_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" + nix path-info --all > /tmp/nix-store-before + ${extra_setup} + export NIX_CONF=" + extra-trusted-public-keys = $NIX_PUBLIC_KEYS \n + extra-trusted-substituters = $NIX_SUBSTITUTERS \n + extra-substituters = $NIX_SUBSTITUTERS \n + $NIX_EXTRA_CONF + " + echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + + echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job" + nix build .#gitlab-ci-job-deps:$1 + source $(readlink -f result) + echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" + ''; + finalizeScript = push_command: + pkgs.writeShellScriptBin "finalize_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache" + nix path-info --all > /tmp/nix-store-after + ${pkgs.diffutils}/bin/diff --new-line-format="%L" \ + --old-line-format="" --unchanged-line-format="" \ + /tmp/nix-store-before /tmp/nix-store-after | ${push_command} + echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" + ''; + mkImage = extraPackages: + pkgs.dockerTools.buildImage { + name = "nix-gitlab-ci"; + fromImage = pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-flakes"; + imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; + sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; + finalImageName = "nixpkgs/nix-flakes"; + finalImageTag = "latest"; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [pkgs.gitMinimal] ++ extraPackages; + pathsToLink = ["/bin"]; + }; }; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = [pkgs.gitMinimal pkgs.cachix inputs'.attic.packages.attic-client]; - pathsToLink = ["/bin"]; - }; - }; + in { + image = mkImage [ + (setupScript '' + echo "No caching configured, to enable caching use the respective container image tag" + '') + (finalizeScript ''${pkgs.busybox}/bin/wc -l | { read count; echo "No caching configured, not uploading $count new store entries..."; }'') + ]; + image-cachix = mkImage [ + (setupScript '' + echo "Configuring caching with cachix..." + ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true + '') + (finalizeScript "${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true") + ]; + image-attic = mkImage [ + (setupScript '' + echo "Configuring caching with attic..." + ${inputs'.attic.packages.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true + ${inputs'.attic.packages.attic-client}/bin/attic use "$ATTIC_CACHE" || true + '') + (finalizeScript "${inputs'.attic.packages.attic-client}/bin/attic push ci:$ATTIC_CACHE || true") + ]; }; + + checks = packages; }; }; @@ -99,7 +150,7 @@ inputs.pre-commit-hooks.follows = "pre-commit-hooks"; }; pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; - attic.url = "github:zhaofengli/attic"; + attic.url = "gitlab:TECHNOFAB/attic"; }; nixConfig = { diff --git a/flakeModule.nix b/flakeModule.nix index 6ef2e09..4145a21 100644 --- a/flakeModule.nix +++ b/flakeModule.nix @@ -123,13 +123,15 @@ config.packages = let toYaml = (pkgs.formats.yaml {}).generate; mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); - prependToBeforeScript = arr: job: + prepend = key: arr: job: job // lib.optionalAttrs job.nix { - before_script = + ${key} = arr - ++ job.before_script or []; + ++ job.${key} or []; }; + prependToBeforeScript = prepend "before_script"; + prependToAfterScript = prepend "after_script"; jobs = filterAttrsRec (n: v: v != null) config.ci.jobs; rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]); @@ -158,10 +160,11 @@ name = key; value = builtins.removeAttrs ( (prependToBeforeScript [ - "echo -e \"\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KDownload Nix deps for job\"" - "nix build .#gitlab-ci-job-deps:${key}" - "source $(readlink -f result)" - "echo -e \"\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K\"" + "source setup_nix_ci ${key}" + ] + job) + // (prependToAfterScript [ + "finalize_nix_ci" ] job) // lib.optionalAttrs job.nix { diff --git a/gitlab-ci.yml b/gitlab-ci.yml index 75913ec..a96107b 100644 --- a/gitlab-ci.yml +++ b/gitlab-ci.yml @@ -1,7 +1,7 @@ variables: - # cachix | attic - NIX_CI_CACHE_TYPE: cachix + # latest | latest-cachix | latest-attic etc. + NIX_CI_IMAGE_TAG: latest stages: - build @@ -9,10 +9,7 @@ stages: nix-ci:build: stage: build - image: nixos/nix:latest - before_script: - - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf - - echo "accept-flake-config = true" >> /etc/nix/nix.conf + image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${NIX_CI_IMAGE_TAG} script: # build the generated-gitlab-ci.yml - nix build .#gitlab-ci-config