Merge branch 'fix-job-patch' into 'main'

fix: weird jobs patching

See merge request TECHNOFAB/nix-gitlab-ci!15

.gitlab-ci.yml

@@ -55,3 +55,5 @@ stages:
 - build-images
 - build
 - trigger
+variables:
+  NIX_CI_IMAGE: $CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA

lib/impl/helpers.nix

@@ -50,7 +50,7 @@ in rec {
   filterJobVariables = shouldContain: job:
     concatMapAttrs (
       name: value:
-        optionalAttrs ((hasInfix "/nix/store/" value) == shouldContain) {
+        optionalAttrs ((hasInfix builtins.storeDir value) == shouldContain) {
           ${name} = value;
         }
     )

lib/impl/jobPatched.nix

@@ -3,7 +3,7 @@
   helpers,
 }: let
   inherit (lib) toList optionalAttrs optional;
-  inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables;
+  inherit (helpers) prependToBeforeScript appendToAfterScript;
 in
   {
     key,
@@ -11,16 +11,13 @@ in
     pipelineName,
     nixConfig,
   }:
-    (builtins.removeAttrs job ["variables" "cache"])
+    if ! nixConfig.enable then job else
-    // (optionalAttrs nixConfig.enable (
+    (builtins.removeAttrs job [ "variables" "cache" ])
-      (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job)
+    // (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\"" ] job)
-      // (appendToAfterScript ["finalize_nix_ci"] job)
+    // (appendToAfterScript [ "finalize_nix_ci" ] job)
-    ))
+    // (let
-    // optionalAttrs nixConfig.enable (
+      variables = job.variables or {} //
-      (let
+        optionalAttrs nixConfig.enableRunnerCache {
-        variables =
-          (filterJobVariables false job)
-          // optionalAttrs nixConfig.enableRunnerCache {
           NIX_CI_CACHE_STRATEGY = "runner";
         };
     in
@@ -29,15 +26,13 @@ in
         inherit variables;
       })
     // (let
-        cache =
+      cache = (toList (job.cache or [])) ++
-          (toList (job.cache or []))
+      (optional nixConfig.enableRunnerCache {
-          ++ (optional nixConfig.enableRunnerCache {
         key = nixConfig.runnerCacheKey;
-            paths = [".nix-cache/"];
+        paths = [ ".nix-cache/" ];
       });
     in
       # filter empty cache
       optionalAttrs (cache != []) {
         inherit cache;
       })
-    )

nix/repo/ci.nix

@@ -10,6 +10,7 @@ in
       # the child pipeline can then use the built images to test them
       extraData = {
         stages = ["build-images" "build" "trigger"];
+        variables.NIX_CI_IMAGE = "$CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA";
         "build:image" = {
           stage = "build-images";
           parallel.matrix = [

tests/cilib_test.nix

@@ -56,24 +56,59 @@
       }
       {
         name = "jobPatched nix disabled";
-        expected = {};
+        expected = {
+          cache = {
+            name = "some";
+            paths = [ "this" ];
+          };
+          variables = {
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
+        };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig.enable = false;
         };
       }
       {
         name = "jobPatched without runner cache";
         expected = {
-          after_script = ["finalize_nix_ci"];
+          after_script = [ "finalize_nix_ci" ];
-          before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
+          before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ];
+          variables = {
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
+          cache = [{
+            name = "some";
+            paths = [ "this" ];
+          }];
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig = {
             enable = true;
             enableRunnerCache = false;
@@ -83,20 +118,34 @@
       {
         name = "jobPatched with runner cache";
         expected = {
-          after_script = ["finalize_nix_ci"];
+          after_script = [ "finalize_nix_ci" ];
-          before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
+          before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ];
-          cache = [
+          cache = [{
-            {
+            name = "some";
+            paths = [ "this" ];
+          }{
             key = "test";
-              paths = [".nix-cache/"];
+            paths = [ ".nix-cache/" ];
-            }
+          }];
-          ];
+          variables = {
-          variables."NIX_CI_CACHE_STRATEGY" = "runner";
+            NIX_CI_CACHE_STRATEGY = "runner";
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig = {
             enable = true;
             enableRunnerCache = true;
@@ -129,7 +178,7 @@
           # sh
           ''
             set -euo pipefail
-            ${ntlib.helpers.path [pkgs.jq pkgs.gnugrep pkgs.coreutils]}
+            ${ntlib.helpers.path (with pkgs; [jq gnugrep coreutils])}
             echo "two keys, one json one pretty"
             jq 'keys | length == 2' "${pipeline}" | grep -q true
             echo "key[0] is exactly 'gitlab-ci:pipeline:test'"
@@ -145,20 +194,107 @@
           '';
       }
       {
-        name = "handle store paths in variables";
+        name = "handle store paths in variables without nix config";
         expected = {
-          stages = ["test"];
+          stages = [ "test" ];
-          test.stage = "test";
+          variables = {
+            EXAMPLE = "empty";
+            CURL = toString pkgs.curl;
           };
-        actual =
+          test = {
-          (mkPipeline {
+            stage = "test";
+            before_script = [ "./init" ];
+            script = [ "echo Hello World!" ];
+            after_script = [ "./clean" ];
+            cache = {
+              key = "simple";
+              paths = [ "~/random/" ];
+            };
+            variables = {
+              SAMPLE = "working";
+              HELLO = toString pkgs.hello;
+            };
+          };
+        };
+        actual = (mkPipeline {
           name = "test";
           nixConfig.enable = false;
           pipeline = {
-              stages = ["test"];
+            stages = [ "test" ];
+            variables = {
+              EXAMPLE = "empty";
+              CURL = toString pkgs.curl;
+            };
             jobs.test = {
               stage = "test";
-                variables."TEST" = "${pkgs.hello}";
+              before_script = [ "./init" ];
+              script = [ "echo Hello World!" ];
+              after_script = [ "./clean" ];
+              cache = {
+                key = "simple";
+                paths = [ "~/random/" ];
+              };
+              variables = {
+                SAMPLE = "working";
+                HELLO = toString pkgs.hello;
+              };
+            };
+          };
+        }).finalConfig;
+      }
+      {
+        name = "handle store paths in variables with nix config";
+        expected = {
+          stages = [ "test" ];
+          variables = {
+            EXAMPLE = "empty";
+            CURL = toString pkgs.curl;
+          };
+          test = {
+            stage = "test";
+            before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init" ];
+            script = [ "echo Hello World!" ];
+            after_script = [ "./clean" "finalize_nix_ci" ];
+            cache = [{
+              key = "simple";
+              paths = [ "~/random/" ];
+            }{
+              key = "random";
+              paths = [ ".nix-cache/" ];
+            }];
+            variables = {
+              SAMPLE = "working";
+              HELLO = toString pkgs.hello;
+              NIX_CI_CACHE_STRATEGY = "runner";
+            };
+          };
+        };
+        actual = (mkPipeline {
+          name = "test";
+          nixConfig = {
+            enable = true;
+            enableRunnerCache = true;
+            runnerCacheKey = "random";
+          };
+          pipeline = {
+            stages = [ "test" ];
+            variables = {
+              EXAMPLE = "empty";
+              CURL = toString pkgs.curl;
+            };
+            jobs.test = {
+              stage = "test";
+              before_script = [ "./init" ];
+              script = [ "echo Hello World!" ];
+              after_script = [ "./clean" ];
+              cache = {
+                key = "simple";
+                paths = [ "~/random/" ];
+              };
+              variables = {
+                SAMPLE = "working";
+                HELLO = toString pkgs.hello;
+              };
             };
           };
         }).finalConfig;