Merge branch 'feat/improve-caching' into 'main'

feat: add support for Gitlab CI cache

Closes #2

See merge request TECHNOFAB/nix-gitlab-ci!4

README.md

@@ -1,9 +1,10 @@
 # Nix Gitlab CI
 
 Flake module which allows generating a `.gitlab-ci.yml` from Nix.
+
 This allows easily using any Nix package in CI.
-Also makes it possible to split CI parts in a separate module
+
-which can be imported in multiple projects.
+Also makes it possible to split CI parts in a separate module which can be imported in multiple projects.
 
 ## Usage
 
@@ -25,7 +26,7 @@ which can be imported in multiple projects.
         jobs = {
           "test" = {
             stage = "test";
-            deps = [pkgs.unixtools.ping];
+            nix.deps = [pkgs.unixtools.ping];
             script = [
               "ping -c 5 8.8.8.8"
             ];
@@ -47,3 +48,16 @@ include:
       image_tag: latest-cachix
 ```
 
+## Utilities
+
+### Disable Caching temporarily
+
+To disable any of the provided caches for a pipeline one can set `NIX_CI_DISABLE_CACHE` to
+anything non-empty (eg. "yes") when triggering the pipeline.
+
+The `build:nix-ci` job has a different special environment variable `NIX_CI_SKIP_CACHE`
+(useful if the generated pipeline is outdated but caching should generally still take place).
+
+## Thanks to
+
+Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix

flake.nix

@@ -53,7 +53,10 @@
               # wait an hour so the image builds
               when = "delayed";
               start_in = "1 hour";
-              deps = [pkgs.hello pkgs.curl];
+              nix = {
+                deps = [pkgs.hello pkgs.curl];
+                disable-cache = false;
+              };
               variables = {
                 TEST = "test";
                 TEST_WITH_DERIVATION = "${pkgs.hello}/test";
@@ -65,7 +68,7 @@
               ];
             };
             "test-non-nix" = {
-              nix = false;
+              nix.enable = false;
               stage = "test";
               image = "alpine:latest";
               script = [
@@ -80,12 +83,18 @@
             pkgs.writeShellScriptBin "setup_nix_ci" ''
               echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI"
                 nix path-info --all > /tmp/nix-store-before
-                ${extra_setup}
+
-                export NIX_CONF="
+                if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
-                  extra-trusted-public-keys = $NIX_PUBLIC_KEYS \n
+                  ${extra_setup}
-                  extra-trusted-substituters = $NIX_SUBSTITUTERS \n
+                else
-                  extra-substituters = $NIX_SUBSTITUTERS \n
+                  echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration"
-                  $NIX_EXTRA_CONF
+                fi
+
+                export NIX_CONFIG="
+                  extra-trusted-public-keys = $NIX_PUBLIC_KEYS
+                  extra-trusted-substituters = $NIX_SUBSTITUTERS
+                  extra-substituters = $NIX_SUBSTITUTERS
+                  $NIX_EXTRA_CONFIG
                 "
               echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"
               ${
@@ -104,7 +113,14 @@
               nix path-info --all > /tmp/nix-store-after
               ${pkgs.diffutils}/bin/diff --new-line-format="%L" \
                 --old-line-format="" --unchanged-line-format="" \
-                /tmp/nix-store-before /tmp/nix-store-after | ${push_command}
+                /tmp/nix-store-before /tmp/nix-store-after \
+              | {
+                if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
+                  ${push_command}
+                else
+                  ${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; }
+                fi
+              }
               echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
             '';
           mkImage = extraPackages:
@@ -129,13 +145,19 @@
               };
             };
         in {
-          setup-script = setupScript "# extra_setup";
+          setup-script = setupScript "true # extra_setup";
           finalize-script = finalizeScript "true # push_command";
           image = mkImage [
             (setupScript ''
-              echo "No caching configured, to enable caching use the respective container image tag"
+              cachedir="$(pwd)/.nix-cache"
+              echo "Configuring caching with the Runner Cache in $cachedir..."
+              export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true"
+            '')
+            (finalizeScript ''
+              while read entry; do
+                nix copy --quiet --to "file://$(pwd)/.nix-cache" $entry || true
+              done
             '')
-            (finalizeScript ''${pkgs.busybox}/bin/wc -l | { read count; echo "No caching configured, not uploading $count new store entries..."; }'')
           ];
           image-cachix = mkImage [
             (setupScript ''

lib/flakeModule.nix

@@ -33,19 +33,46 @@
           nix-jobs-per-default = mkOption {
             type = types.bool;
             default = true;
-            description = "Handle jobs nix-based by default or via opt-in (in job set nix = true) if false";
+            description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
+          };
+          disable-cache = mkOption {
+            type = types.bool;
+            default = false;
+            description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE";
+          };
+          cache-key = mkOption {
+            type = types.str;
+            default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
+            description = "Cache key to use for the nix cache";
           };
         };
       jobType = with lib;
         subType {
           # nix ci opts
           nix = mkOption {
-            type = types.bool;
+            type = subType {
-            default = cfg.nix-jobs-per-default;
+              enable = mkOption {
-          };
+                type = types.bool;
-          deps = mkOption {
+                default = cfg.nix-jobs-per-default;
-            type = types.listOf types.package;
+                description = "Handle this job as a nix job";
-            default = [];
+              };
+              deps = mkOption {
+                type = types.listOf types.package;
+                default = [];
+                description = "Dependencies/packages to install for this job";
+              };
+              disable-cache = mkOption {
+                type = types.bool;
+                default = cfg.disable-cache;
+                description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE";
+              };
+              cache-key = mkOption {
+                type = types.str;
+                default = cfg.cache-key;
+                description = "Cache key to use for the nix cache";
+              };
+            };
+            description = "Configure Nix Gitlab CI for each job individually";
           };
           # gitlab opts
           script = mkOption {
@@ -64,7 +91,7 @@
           allow_failure = mkNullOption (types.either types.attrs types.bool);
           artifacts = mkNullOption (types.attrs);
           before_script = mkNullOption (types.listOf types.str);
-          cache = mkNullOption (types.attrs);
+          cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
           coverage = mkNullOption (types.str);
           dependencies = mkNullOption (types.listOf types.str);
           environment = mkNullOption (types.either types.attrs types.str);
@@ -124,7 +151,7 @@
         mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
         prepend = key: arr: job:
           job
-          // lib.optionalAttrs job.nix {
+          // lib.optionalAttrs job.nix.enable {
             ${key} =
               arr
               ++ job.${key} or [];
@@ -143,11 +170,9 @@
             variablesWithStorePaths =
               lib.concatMapAttrs (
                 name: value:
-                  if lib.hasInfix "/nix/store/" value
+                  lib.optionalAttrs (lib.hasInfix "/nix/store/" value) {
-                  then {
                     ${name} = value;
                   }
-                  else {}
               )
               (job.variables or {});
             variableExports = lib.concatMapStrings (x: "${x}\n") (
@@ -156,7 +181,7 @@
           in {
             name = "gitlab-ci-job-deps:${key}";
             value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" ''
-              export PATH="${lib.makeBinPath job.deps}:$PATH";
+              export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
               ${variableExports}
             '';
           })
@@ -181,17 +206,32 @@
                       "finalize_nix_ci"
                     ]
                     job))
-                // lib.optionalAttrs job.nix {
+                // lib.optionalAttrs job.nix.enable {
                   image = job.image;
-                  variables = lib.concatMapAttrs (name: value:
+                  variables =
-                    if lib.hasInfix "/nix/store/" value
+                    lib.concatMapAttrs (name: value:
-                    then {}
+                      lib.optionalAttrs (!lib.hasInfix "/nix/store/" value) {
-                    else {
+                        ${name} = value;
-                      ${name} = value;
+                      })
-                    })
+                    (job.variables or {})
-                  (job.variables or {});
+                    // lib.optionalAttrs job.nix.disable-cache {
+                      NIX_CI_DISABLE_CACHE = "yes";
+                    };
+                  cache =
+                    (
+                      let
+                        c = job.cache or [];
+                      in
+                        if builtins.isList c
+                        then c
+                        else [c]
+                    )
+                    ++ (lib.optional (!job.nix.disable-cache) {
+                      key = job.nix.cache-key;
+                      paths = [".nix-cache/"];
+                    });
                 }
-              ) ["nix" "deps"];
+              ) ["nix"];
           })
           jobs;
       in

templates/nix-gitlab-ci.yml

@@ -4,28 +4,53 @@ spec:
       type: string
       description: "latest | latest-cachix | latest-attic etc."
       default: latest
+    cache_files:
+      type: array
+      description: |
+        Files to use as the cache key for the generated pipeline yaml.
+        If you use "ci.nix" to define CI, add that here for example
+      default: ["flake.nix", "flake.lock"]
+    disable_cache:
+      type: string
+      description: |
+        Disables any caching provided by this component. Set to any non-empty value to disable caching.
+      default: ""
 ---
-
 stages:
   - build
   - trigger
-
+variables:
+  NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]"
 nix-ci:build:
   stage: build
   image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]]
+  cache:
+    - key:
+        files: $[[ inputs.cache_files ]]
+      paths:
+        - generated-gitlab-ci.yml
+    - key: nix
+      paths:
+        - .nix-cache/
   before_script:
-    - source setup_nix_ci
+    # generated-gitlab-ci.yml exists in the cache
+    - '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "Using cached pipeline file (skip cache with NIX_CI_SKIP_CACHE)" || true'
+    # allow the user to manually skip the cache (when the key files are not correctly configured etc.)
+    - '[ -n "$NIX_CI_SKIP_CACHE" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_SKIP_CACHE)" || true'
+    # only setup when we need to generate the pipeline yaml
+    - 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi'
   script:
-    # build the generated-gitlab-ci.yml
+    # build the generated-gitlab-ci.yml if it does not exist in the cache
-    - nix build .#gitlab-ci-config
+    - 'if [ -z "$CACHED" ]; then nix build .#gitlab-ci-config && install result generated-gitlab-ci.yml; fi'
-    - install result generated-gitlab-ci.yml
   after_script:
-    # upload to binary cache
+    # NOTE: environment variables of before_script and script don't exist here anymore
-    - finalize_nix_ci
+    #
+    # save to binary cache or Gitlab CI cache only if we actually built something
+    #  check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called
+    - 'if [ -f "/tmp/nix-store-before" ]; then finalize_nix_ci; fi'
   artifacts:
     paths:
       - generated-gitlab-ci.yml
-
 nix-ci:trigger:
   stage: trigger
   needs:
@@ -35,4 +60,5 @@ nix-ci:trigger:
       - artifact: generated-gitlab-ci.yml
         job: nix-ci:build
     strategy: depend
-
+    forward:
+      pipeline_variables: true