lib/impl/jobPatched.nix

@@ -3,7 +3,7 @@
   helpers,
 }: let
   inherit (lib) toList optionalAttrs optional;
-  inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables;
+  inherit (helpers) prependToBeforeScript appendToAfterScript;
 in
   {
     key,
@@ -11,32 +11,28 @@ in
     pipelineName,
     nixConfig,
   }:
-    if ! nixConfig.enable
+    if ! nixConfig.enable then job else
-    then job
+    (builtins.removeAttrs job [ "variables" "cache" ])
-    else
+    // (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\"" ] job)
-      (builtins.removeAttrs job ["variables" "cache"])
+    // (appendToAfterScript [ "finalize_nix_ci" ] job)
-      // (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job)
+    // (let
-      // (appendToAfterScript ["finalize_nix_ci"] job)
+      variables = job.variables or {} //
-      // (let
+        optionalAttrs nixConfig.enableRunnerCache {
-        variables =
+          NIX_CI_CACHE_STRATEGY = "runner";
-          (filterJobVariables false job)
+        };
-          // optionalAttrs nixConfig.enableRunnerCache {
+    in
-            NIX_CI_CACHE_STRATEGY = "runner";
+      # filter empty variables
-          };
+      optionalAttrs (variables != {}) {
-      in
+        inherit variables;
-        # filter empty variables
+      })
-        optionalAttrs (variables != {}) {
+    // (let
-          inherit variables;
+      cache = (toList (job.cache or [])) ++
-        })
+      (optional nixConfig.enableRunnerCache {
-      // (let
+        key = nixConfig.runnerCacheKey;
-        cache =
+        paths = [ ".nix-cache/" ];
-          (toList (job.cache or []))
+      });
-          ++ (optional nixConfig.enableRunnerCache {
+    in
-            key = nixConfig.runnerCacheKey;
+      # filter empty cache
-            paths = [".nix-cache/"];
+      optionalAttrs (cache != []) {
-          });
+        inherit cache;
-      in
+      })
-        # filter empty cache
-        optionalAttrs (cache != []) {
-          inherit cache;
-        })

lib/impl/modules/job.nix

@@ -646,7 +646,7 @@ in rec {
       };
       depsDrv = cilib.mkJobDeps {
         key = name;
-        inherit job;
+        job = config.finalConfig;
         nixConfig = config.nix;
       };
       runnerDrv = cilib.mkJobRun {

nix/repo/ci.nix

@@ -1,5 +1,5 @@
 {inputs, ...}: let
-  inherit (inputs) cilib pkgs;
+  inherit (inputs) cilib;
 in
   cilib.mkCI {
     config.soonix = {
@@ -58,25 +58,9 @@ in
       };
     };
     pipelines."default" = {
-      stages = ["check" "test" "build" "deploy"];
+      stages = ["test" "build" "deploy"];
-      variables = {
-        EXAMPLE = "empty";
-        CURL = toString pkgs.curl;
-      };
       jobs = {
-        "check" = {
-          stage = "check";
-          script = [
-            "set -euo pipefail"
-            "echo EXAMPLE=$EXAMPLE CURL=$CURL SAMPLE=$SAMPLE HELLO=$HELLO"
-          ];
-          variables = {
-            SAMPLE = "working";
-            HELLO = toString pkgs.hello;
-          };
-        };
         "test" = {
-          nix.deps = with pkgs; [coreutils nix];
           stage = "test";
           script = [
             "nix run .#tests -- --junit=junit.xml"
@@ -89,7 +73,6 @@ in
         };
         "docs" = {
           stage = "build";
-          nix.deps = with pkgs; [coreutils nix];
           script = [
             # sh
             ''

tests/cilib_test.nix

@@ -56,26 +56,28 @@
       }
       {
         name = "jobPatched nix disabled";
-        expected = {};
-        actual = mkJobPatched {
-          key = "test";
-          pipelineName = "test";
-          job = {};
-          nixConfig.enable = false;
-        };
-      }
-      {
-        name = "jobPatched nix disabled with variables and cache";
         expected = {
-          variables."HELLO" = "world";
+          cache = {
-          cache = [{key = "example";}];
+            name = "some";
+            paths = [ "this" ];
+          };
+          variables = {
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
           job = {
-            variables."HELLO" = "world";
+            cache = {
-            cache = [{key = "example";}];
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
           };
           nixConfig.enable = false;
         };
@@ -83,13 +85,30 @@
       {
         name = "jobPatched without runner cache";
         expected = {
-          after_script = ["finalize_nix_ci"];
+          after_script = [ "finalize_nix_ci" ];
-          before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
+          before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ];
+          variables = {
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
+          cache = [{
+            name = "some";
+            paths = [ "this" ];
+          }];
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig = {
             enable = true;
             enableRunnerCache = false;
@@ -99,20 +118,34 @@
       {
         name = "jobPatched with runner cache";
         expected = {
-          after_script = ["finalize_nix_ci"];
+          after_script = [ "finalize_nix_ci" ];
-          before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
+          before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ];
-          cache = [
+          cache = [{
-            {
+            name = "some";
-              key = "test";
+            paths = [ "this" ];
-              paths = [".nix-cache/"];
+          }{
-            }
+            key = "test";
-          ];
+            paths = [ ".nix-cache/" ];
-          variables."NIX_CI_CACHE_STRATEGY" = "runner";
+          }];
+          variables = {
+            NIX_CI_CACHE_STRATEGY = "runner";
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig = {
             enable = true;
             enableRunnerCache = true;
@@ -161,48 +194,110 @@
           '';
       }
       {
-        name = "ignore store paths in variables with nix disabled";
+        name = "handle store paths in variables without nix config";
-        expected = {
-          stages = ["test"];
-          test = {
-            stage = "test";
-            variables."TEST" = "${pkgs.hello}";
-          };
-        };
-        actual =
-          (mkPipeline {
-            name = "test";
-            nixConfig.enable = false;
-            pipeline = {
-              stages = ["test"];
-              jobs.test = {
-                stage = "test";
-                variables."TEST" = "${pkgs.hello}";
-              };
-            };
-          }).finalConfig;
-      }
-      {
-        # it doesn't make much sense to have any nix store path in variables, but we ignore it for global variables
-        name = "ignore store paths in global variables";
         expected = {
+          stages = [ "test" ];
           variables = {
-            HELLO = "world";
+            EXAMPLE = "empty";
             CURL = toString pkgs.curl;
           };
-        };
+          test = {
-        actual =
+            stage = "test";
-          (mkPipeline {
+            before_script = [ "./init" ];
-            name = "test";
+            script = [ "echo Hello World!" ];
-            nixConfig.enable = true;
+            after_script = [ "./clean" ];
-            pipeline = {
+            cache = {
-              variables = {
+              key = "simple";
-                HELLO = "world";
+              paths = [ "~/random/" ];
-                CURL = toString pkgs.curl;
-              };
-              jobs = {};
             };
-          }).finalConfig;
+            variables = {
+              SAMPLE = "working";
+              HELLO = toString pkgs.hello;
+            };
+          };
+        };
+        actual = (mkPipeline {
+          name = "test";
+          nixConfig.enable = false;
+          pipeline = {
+            stages = [ "test" ];
+            variables = {
+              EXAMPLE = "empty";
+              CURL = toString pkgs.curl;
+            };
+            jobs.test = {
+              stage = "test";
+              before_script = [ "./init" ];
+              script = [ "echo Hello World!" ];
+              after_script = [ "./clean" ];
+              cache = {
+                key = "simple";
+                paths = [ "~/random/" ];
+              };
+              variables = {
+                SAMPLE = "working";
+                HELLO = toString pkgs.hello;
+              };
+            };
+          };
+        }).finalConfig;
+      }
+      {
+        name = "handle store paths in variables with nix config";
+        expected = {
+          stages = [ "test" ];
+          variables = {
+            EXAMPLE = "empty";
+            CURL = toString pkgs.curl;
+          };
+          test = {
+            stage = "test";
+            before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init" ];
+            script = [ "echo Hello World!" ];
+            after_script = [ "./clean" "finalize_nix_ci" ];
+            cache = [{
+              key = "simple";
+              paths = [ "~/random/" ];
+            }{
+              key = "random";
+              paths = [ ".nix-cache/" ];
+            }];
+            variables = {
+              SAMPLE = "working";
+              HELLO = toString pkgs.hello;
+              NIX_CI_CACHE_STRATEGY = "runner";
+            };
+          };
+        };
+        actual = (mkPipeline {
+          name = "test";
+          nixConfig = {
+            enable = true;
+            enableRunnerCache = true;
+            runnerCacheKey = "random";
+          };
+          pipeline = {
+            stages = [ "test" ];
+            variables = {
+              EXAMPLE = "empty";
+              CURL = toString pkgs.curl;
+            };
+            jobs.test = {
+              stage = "test";
+              before_script = [ "./init" ];
+              script = [ "echo Hello World!" ];
+              after_script = [ "./clean" ];
+              cache = {
+                key = "simple";
+                paths = [ "~/random/" ];
+              };
+              variables = {
+                SAMPLE = "working";
+                HELLO = toString pkgs.hello;
+              };
+            };
+          };
+        }).finalConfig;
       }
     ];
   };

tests/modules_test.nix

@@ -10,17 +10,9 @@
       simplePipeline = cilib.mkCI {
         pipelines."test" = {
           stages = ["test"];
-          variables = {
-            EXAMPLE = "empty";
-            CURL = toString pkgs.curl;
-          };
           jobs."test" = {
             stage = "test";
             script = ["echo hello world"];
-            variables = {
-              SAMPLE = "working";
-              HELLO = toString pkgs.hello;
-            };
           };
         };
       };
@@ -38,29 +30,21 @@
           (cilib.mkCI {}).packages;
       }
       {
-        name = "simple pipeline final config";
+        name = "simple pipeline";
         expected = {
           stages = [".pre" "test" ".post"];
-          variables = {
-            EXAMPLE = "empty";
-            CURL = toString pkgs.curl;
-          };
           "test" = {
             image = "$NIX_CI_IMAGE";
             stage = "test";
             before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
             script = ["echo hello world"];
             after_script = ["finalize_nix_ci"];
-            variables = {
-              SAMPLE = "working";
-              #HELLO = toString pkgs.hello;
-            };
           };
         };
         actual = simplePipeline.pipelines."test".finalConfig;
       }
       {
-        name = "simple pipeline json";
+        name = "simple pipeline yaml";
         type = "script";
         script = let
           package = simplePipeline.packages."gitlab-ci:pipeline:test";
@@ -72,23 +56,25 @@
             assert_file_contains ${package} 'gitlab-ci:pipeline:test:job-deps:test'
             assert_file_contains ${package} 'finalize_nix_ci'
             assert_file_contains ${package} 'echo hello world'
-            assert_file_contains ${package} '"EXAMPLE":"empty"'
-            assert_file_contains ${package} '"SAMPLE":"working"'
-            assert_file_contains ${package} '"CURL":"/nix/store/.*-curl-.*"'
           '';
       }
       {
-        name = "simple pipeline deps drv";
+        name = "dont fail on store paths";
         type = "script";
         script = let
-          package = simplePipeline.packages."gitlab-ci:pipeline:test:job-deps:test";
+          package =
+            (cilib.mkCI {
+              pipelines."test" = {
+                variables.EXAMPLE = "${pkgs.hello}";
+              };
+            }).packages."gitlab-ci:pipeline:test";
         in
           # sh
           ''
             ${ntlib.helpers.path [pkgs.gnugrep]}
             ${ntlib.helpers.scriptHelpers}
-            assert_file_contains ${package} ':$PATH'
+            assert_file_contains ${package} '[".pre",".post"]'
-            assert_file_contains ${package} 'HELLO="/nix/store/.*-hello-.*"'
+            assert_file_contains ${package} '"EXAMPLE":"/nix/store/.*-hello-.*"'
           '';
       }
     ];