feat: caching support

.gitlab-ci.yml

@@ -7,16 +7,18 @@ stages:
 
 build:image:
   stage: build
+  parallel:
+    matrix:
+      - VARIANT: [ "", "-cachix", "-attic" ]
   image: nixpkgs/nix-flakes:latest
   before_script:
     - nix profile install nixpkgs#skopeo
     - export PATH="$PATH:$HOME/.nix-profile/bin"
-    - mkdir -p /var/tmp
   script:
-    - nix build .#image
+    - nix build .#image${VARIANT}
-    - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}"
+    - skopeo --insecure-policy copy  --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp
       "docker-archive:result"
-      "docker://$CI_REGISTRY_IMAGE/nix-ci:latest"
+      "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_BRANCH/main/latest}${VARIANT}"
-    - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}"
+    - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp
       "docker-archive:result"
-      "docker://$CI_REGISTRY_IMAGE/nix-ci:$(date +"%m-%d-%y")"
+      "docker://$CI_REGISTRY_IMAGE/nix-ci:$(date +"%m-%d-%y")${VARIANT}"

flake.lock

@@ -9,17 +9,17 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1704159259,
+        "lastModified": 1710680783,
-        "narHash": "sha256-OOEFibN9JQBepVeqxSNSBr6JUmdoZiH263ogms2jk0k=",
+        "narHash": "sha256-exrsUUk/VSCG2Y3Sr/dkST8vwICzab3fe8Je81t/SfM=",
-        "owner": "zhaofengli",
+        "owner": "TECHNOFAB",
         "repo": "attic",
-        "rev": "e6bedf1869f382cfc51b69848d6e09d51585ead6",
+        "rev": "b6d8d5ca53ef15910042a39e81f35383370c1112",
-        "type": "github"
+        "type": "gitlab"
       },
       "original": {
-        "owner": "zhaofengli",
+        "owner": "TECHNOFAB",
         "repo": "attic",
-        "type": "github"
+        "type": "gitlab"
       }
     },
     "crane": {

flake.nix

@@ -24,11 +24,11 @@
         system,
         inputs',
         ...
-      }: {
+      }: rec {
         formatter = pkgs.alejandra;
-        devenv.shells = {
+        devenv.shells.default = {
-          default = {
+          containers = pkgs.lib.mkForce {};
-            packages = [pkgs.dive pkgs.skopeo];
+          packages = with pkgs; [dive skopeo];
 
           pre-commit = {
             hooks = {
@@ -36,7 +36,6 @@
             };
           };
         };
-        };
         ci = {
           stages = ["test"];
           default = {
@@ -68,8 +67,36 @@
           };
         };
 
-        packages = {
+        packages = let
-          image = pkgs.dockerTools.buildImage {
+          setupScript = extra_setup:
+            pkgs.writeShellScriptBin "setup_nix_ci" ''
+              echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI"
+              nix path-info --all > /tmp/nix-store-before
+              ${extra_setup}
+              export NIX_CONF="
+                extra-trusted-public-keys = $NIX_PUBLIC_KEYS \n
+                extra-trusted-substituters = $NIX_SUBSTITUTERS \n
+                extra-substituters = $NIX_SUBSTITUTERS \n
+                $NIX_EXTRA_CONF
+              "
+              echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"
+
+              echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job"
+              nix build .#gitlab-ci-job-deps:$1
+              source $(readlink -f result)
+              echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K"
+            '';
+          finalizeScript = push_command:
+            pkgs.writeShellScriptBin "finalize_nix_ci" ''
+              echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache"
+              nix path-info --all > /tmp/nix-store-after
+              ${pkgs.diffutils}/bin/diff --new-line-format="%L" \
+                --old-line-format="" --unchanged-line-format="" \
+                /tmp/nix-store-before /tmp/nix-store-after | ${push_command}
+              echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
+            '';
+          mkImage = extraPackages:
+            pkgs.dockerTools.buildImage {
               name = "nix-gitlab-ci";
               fromImage = pkgs.dockerTools.pullImage {
                 imageName = "nixpkgs/nix-flakes";
@@ -80,11 +107,35 @@
               };
               copyToRoot = pkgs.buildEnv {
                 name = "image-root";
-              paths = [pkgs.gitMinimal pkgs.cachix inputs'.attic.packages.attic-client];
+                paths = [pkgs.gitMinimal] ++ extraPackages;
                 pathsToLink = ["/bin"];
               };
             };
+        in {
+          image = mkImage [
+            (setupScript ''
+              echo "No caching configured, to enable caching use the respective container image tag"
+            '')
+            (finalizeScript ''${pkgs.busybox}/bin/wc -l | { read count; echo "No caching configured, not uploading $count new store entries..."; }'')
+          ];
+          image-cachix = mkImage [
+            (setupScript ''
+              echo "Configuring caching with cachix..."
+              ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true
+            '')
+            (finalizeScript "${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true")
+          ];
+          image-attic = mkImage [
+            (setupScript ''
+              echo "Configuring caching with attic..."
+              ${inputs'.attic.packages.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true
+              ${inputs'.attic.packages.attic-client}/bin/attic use "$ATTIC_CACHE" || true
+            '')
+            (finalizeScript "${inputs'.attic.packages.attic-client}/bin/attic push ci:$ATTIC_CACHE || true")
+          ];
         };
+
+        checks = packages;
       };
     };
 
@@ -99,7 +150,7 @@
       inputs.pre-commit-hooks.follows = "pre-commit-hooks";
     };
     pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
-    attic.url = "github:zhaofengli/attic";
+    attic.url = "gitlab:TECHNOFAB/attic";
   };
 
   nixConfig = {

flakeModule.nix

@@ -123,13 +123,15 @@
       config.packages = let
         toYaml = (pkgs.formats.yaml {}).generate;
         mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
-        prependToBeforeScript = arr: job:
+        prepend = key: arr: job:
           job
           // lib.optionalAttrs job.nix {
-            before_script =
+            ${key} =
               arr
-              ++ job.before_script or [];
+              ++ job.${key} or [];
           };
+        prependToBeforeScript = prepend "before_script";
+        prependToAfterScript = prepend "after_script";
 
         jobs = filterAttrsRec (n: v: v != null) config.ci.jobs;
         rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]);
@@ -158,10 +160,11 @@
             name = key;
             value = builtins.removeAttrs (
               (prependToBeforeScript [
-                  "echo -e \"\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KDownload Nix deps for job\""
+                  "source setup_nix_ci ${key}"
-                  "nix build .#gitlab-ci-job-deps:${key}"
+                ]
-                  "source $(readlink -f result)"
+                job)
-                  "echo -e \"\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K\""
+              // (prependToAfterScript [
+                  "finalize_nix_ci"
                 ]
                 job)
               // lib.optionalAttrs job.nix {

gitlab-ci.yml

@@ -1,7 +1,7 @@
 
 variables:
-  # cachix | attic
+  # latest | latest-cachix | latest-attic etc.
-  NIX_CI_CACHE_TYPE: cachix
+  NIX_CI_IMAGE_TAG: latest
 
 stages:
   - build
@@ -9,10 +9,7 @@ stages:
 
 nix-ci:build:
   stage: build
-  image: nixos/nix:latest
+  image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${NIX_CI_IMAGE_TAG}
-  before_script:
-    - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
-    - echo "accept-flake-config = true" >> /etc/nix/nix.conf
   script:
     # build the generated-gitlab-ci.yml
     - nix build .#gitlab-ci-config