Merge branch 'fix-job-patch' into 'main'

fix: weird jobs patching

See merge request TECHNOFAB/nix-gitlab-ci!15

docs/index.md

@@ -9,14 +9,3 @@ This project provides a Nix flake module that allows you to generate your `.gitl
 - **Modularity:** Define and manage your CI configurations in a structured and modular way using Nix modules, making it easier to share and reuse CI logic across multiple projects.
 
 This documentation will guide you through setting up and using Nix GitLab CI for your projects.
-
-## Warnings
-
-To save you from frantically searching these docs if something doesn't work as expected, here are the most important warnings ;)
-
-!!! warning
-
-    Do not put Nix store paths into global/pipeline variables. They will simply be passed through,
-    resulting in bad portability (if two runners have different archs for example, one cannot find the path).
-    If you need any Nix store path in env variables, always do it on the job level, there
-    it will automatically be computed at runtime, thus will always work no matter which runner it runs on.

lib/VERSION

@@ -1 +1 @@
-3.1.0
+3.0.1

lib/impl/jobPatched.nix

@@ -3,7 +3,7 @@
   helpers,
 }: let
   inherit (lib) toList optionalAttrs optional;
-  inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables;
+  inherit (helpers) prependToBeforeScript appendToAfterScript;
 in
   {
     key,
@@ -11,32 +11,28 @@ in
     pipelineName,
     nixConfig,
   }:
-    if ! nixConfig.enable
+    if ! nixConfig.enable then job else
-    then job
+    (builtins.removeAttrs job [ "variables" "cache" ])
-    else
+    // (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\"" ] job)
-      (builtins.removeAttrs job ["variables" "cache"])
+    // (appendToAfterScript [ "finalize_nix_ci" ] job)
-      // (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job)
+    // (let
-      // (appendToAfterScript ["finalize_nix_ci"] job)
+      variables = job.variables or {} //
-      // (let
+        optionalAttrs nixConfig.enableRunnerCache {
-        variables =
+          NIX_CI_CACHE_STRATEGY = "runner";
-          (filterJobVariables false job)
+        };
-          // optionalAttrs nixConfig.enableRunnerCache {
+    in
-            NIX_CI_CACHE_STRATEGY = "runner";
+      # filter empty variables
-          };
+      optionalAttrs (variables != {}) {
-      in
+        inherit variables;
-        # filter empty variables
+      })
-        optionalAttrs (variables != {}) {
+    // (let
-          inherit variables;
+      cache = (toList (job.cache or [])) ++
-        })
+      (optional nixConfig.enableRunnerCache {
-      // (let
+        key = nixConfig.runnerCacheKey;
-        cache =
+        paths = [ ".nix-cache/" ];
-          (toList (job.cache or []))
+      });
-          ++ (optional nixConfig.enableRunnerCache {
+    in
-            key = nixConfig.runnerCacheKey;
+      # filter empty cache
-            paths = [".nix-cache/"];
+      optionalAttrs (cache != []) {
-          });
+        inherit cache;
-      in
+      })
-        # filter empty cache
-        optionalAttrs (cache != []) {
-          inherit cache;
-        })

lib/impl/modules/job.nix

@@ -646,7 +646,7 @@ in rec {
       };
       depsDrv = cilib.mkJobDeps {
         key = name;
-        inherit job;
+        job = config.finalConfig;
         nixConfig = config.nix;
       };
       runnerDrv = cilib.mkJobRun {

nix/repo/ci.nix

@@ -1,5 +1,5 @@
 {inputs, ...}: let
-  inherit (inputs) cilib pkgs;
+  inherit (inputs) cilib;
 in
   cilib.mkCI {
     config.soonix = {
@@ -58,25 +58,9 @@ in
       };
     };
     pipelines."default" = {
-      stages = ["check" "test" "build" "deploy"];
+      stages = ["test" "build" "deploy"];
-      variables = {
-        EXAMPLE = "empty";
-        CURL = toString pkgs.curl;
-      };
       jobs = {
-        "check" = {
-          stage = "check";
-          script = [
-            "set -euo pipefail"
-            "echo EXAMPLE=$EXAMPLE CURL=$CURL SAMPLE=$SAMPLE HELLO=$HELLO"
-          ];
-          variables = {
-            SAMPLE = "working";
-            HELLO = toString pkgs.hello;
-          };
-        };
         "test" = {
-          nix.deps = with pkgs; [coreutils nix];
           stage = "test";
           script = [
             "nix run .#tests -- --junit=junit.xml"
@@ -89,7 +73,6 @@ in
         };
         "docs" = {
           stage = "build";
-          nix.deps = with pkgs; [coreutils nix];
           script = [
             # sh
             ''

tests/cilib_test.nix

@@ -56,26 +56,28 @@
       }
       {
         name = "jobPatched nix disabled";
-        expected = {};
-        actual = mkJobPatched {
-          key = "test";
-          pipelineName = "test";
-          job = {};
-          nixConfig.enable = false;
-        };
-      }
-      {
-        name = "jobPatched nix disabled with variables and cache";
         expected = {
-          variables."HELLO" = "world";
+          cache = {
-          cache = [{key = "example";}];
+            name = "some";
+            paths = [ "this" ];
+          };
+          variables = {
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
           job = {
-            variables."HELLO" = "world";
+            cache = {
-            cache = [{key = "example";}];
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
           };
           nixConfig.enable = false;
         };
@@ -83,13 +85,30 @@
       {
         name = "jobPatched without runner cache";
         expected = {
-          after_script = ["finalize_nix_ci"];
+          after_script = [ "finalize_nix_ci" ];
-          before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
+          before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ];
+          variables = {
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
+          cache = [{
+            name = "some";
+            paths = [ "this" ];
+          }];
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig = {
             enable = true;
             enableRunnerCache = false;
@@ -99,20 +118,34 @@
       {
         name = "jobPatched with runner cache";
         expected = {
-          after_script = ["finalize_nix_ci"];
+          after_script = [ "finalize_nix_ci" ];
-          before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
+          before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ];
-          cache = [
+          cache = [{
-            {
+            name = "some";
-              key = "test";
+            paths = [ "this" ];
-              paths = [".nix-cache/"];
+          }{
-            }
+            key = "test";
-          ];
+            paths = [ ".nix-cache/" ];
-          variables."NIX_CI_CACHE_STRATEGY" = "runner";
+          }];
+          variables = {
+            NIX_CI_CACHE_STRATEGY = "runner";
+            BASH = toString pkgs.bash;
+            TEST = "work";
+          };
         };
         actual = mkJobPatched {
           key = "test";
           pipelineName = "test";
-          job = {};
+          job = {
+            cache = {
+              name = "some";
+              paths = [ "this" ];
+            };
+            variables = {
+              BASH = toString pkgs.bash;
+              TEST = "work";
+            };
+          };
           nixConfig = {
             enable = true;
             enableRunnerCache = true;
@@ -161,48 +194,110 @@
           '';
       }
       {
-        name = "ignore store paths in variables with nix disabled";
+        name = "handle store paths in variables without nix config";
-        expected = {
-          stages = ["test"];
-          test = {
-            stage = "test";
-            variables."TEST" = "${pkgs.hello}";
-          };
-        };
-        actual =
-          (mkPipeline {
-            name = "test";
-            nixConfig.enable = false;
-            pipeline = {
-              stages = ["test"];
-              jobs.test = {
-                stage = "test";
-                variables."TEST" = "${pkgs.hello}";
-              };
-            };
-          }).finalConfig;
-      }
-      {
-        # it doesn't make much sense to have any nix store path in variables, but we ignore it for global variables
-        name = "ignore store paths in global variables";
         expected = {
+          stages = [ "test" ];
           variables = {
-            HELLO = "world";
+            EXAMPLE = "empty";
             CURL = toString pkgs.curl;
           };
-        };
+          test = {
-        actual =
+            stage = "test";
-          (mkPipeline {
+            before_script = [ "./init" ];
-            name = "test";
+            script = [ "echo Hello World!" ];
-            nixConfig.enable = true;
+            after_script = [ "./clean" ];
-            pipeline = {
+            cache = {
-              variables = {
+              key = "simple";
-                HELLO = "world";
+              paths = [ "~/random/" ];
-                CURL = toString pkgs.curl;
-              };
-              jobs = {};
             };
-          }).finalConfig;
+            variables = {
+              SAMPLE = "working";
+              HELLO = toString pkgs.hello;
+            };
+          };
+        };
+        actual = (mkPipeline {
+          name = "test";
+          nixConfig.enable = false;
+          pipeline = {
+            stages = [ "test" ];
+            variables = {
+              EXAMPLE = "empty";
+              CURL = toString pkgs.curl;
+            };
+            jobs.test = {
+              stage = "test";
+              before_script = [ "./init" ];
+              script = [ "echo Hello World!" ];
+              after_script = [ "./clean" ];
+              cache = {
+                key = "simple";
+                paths = [ "~/random/" ];
+              };
+              variables = {
+                SAMPLE = "working";
+                HELLO = toString pkgs.hello;
+              };
+            };
+          };
+        }).finalConfig;
+      }
+      {
+        name = "handle store paths in variables with nix config";
+        expected = {
+          stages = [ "test" ];
+          variables = {
+            EXAMPLE = "empty";
+            CURL = toString pkgs.curl;
+          };
+          test = {
+            stage = "test";
+            before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init" ];
+            script = [ "echo Hello World!" ];
+            after_script = [ "./clean" "finalize_nix_ci" ];
+            cache = [{
+              key = "simple";
+              paths = [ "~/random/" ];
+            }{
+              key = "random";
+              paths = [ ".nix-cache/" ];
+            }];
+            variables = {
+              SAMPLE = "working";
+              HELLO = toString pkgs.hello;
+              NIX_CI_CACHE_STRATEGY = "runner";
+            };
+          };
+        };
+        actual = (mkPipeline {
+          name = "test";
+          nixConfig = {
+            enable = true;
+            enableRunnerCache = true;
+            runnerCacheKey = "random";
+          };
+          pipeline = {
+            stages = [ "test" ];
+            variables = {
+              EXAMPLE = "empty";
+              CURL = toString pkgs.curl;
+            };
+            jobs.test = {
+              stage = "test";
+              before_script = [ "./init" ];
+              script = [ "echo Hello World!" ];
+              after_script = [ "./clean" ];
+              cache = {
+                key = "simple";
+                paths = [ "~/random/" ];
+              };
+              variables = {
+                SAMPLE = "working";
+                HELLO = toString pkgs.hello;
+              };
+            };
+          };
+        }).finalConfig;
       }
     ];
   };

tests/modules_test.nix

@@ -10,17 +10,9 @@
       simplePipeline = cilib.mkCI {
         pipelines."test" = {
           stages = ["test"];
-          variables = {
-            EXAMPLE = "empty";
-            CURL = toString pkgs.curl;
-          };
           jobs."test" = {
             stage = "test";
             script = ["echo hello world"];
-            variables = {
-              SAMPLE = "working";
-              HELLO = toString pkgs.hello;
-            };
           };
         };
       };
@@ -38,29 +30,21 @@
           (cilib.mkCI {}).packages;
       }
       {
-        name = "simple pipeline final config";
+        name = "simple pipeline";
         expected = {
           stages = [".pre" "test" ".post"];
-          variables = {
-            EXAMPLE = "empty";
-            CURL = toString pkgs.curl;
-          };
           "test" = {
             image = "$NIX_CI_IMAGE";
             stage = "test";
             before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""];
             script = ["echo hello world"];
             after_script = ["finalize_nix_ci"];
-            variables = {
-              SAMPLE = "working";
-              #HELLO = toString pkgs.hello;
-            };
           };
         };
         actual = simplePipeline.pipelines."test".finalConfig;
       }
       {
-        name = "simple pipeline json";
+        name = "simple pipeline yaml";
         type = "script";
         script = let
           package = simplePipeline.packages."gitlab-ci:pipeline:test";
@@ -72,23 +56,25 @@
             assert_file_contains ${package} 'gitlab-ci:pipeline:test:job-deps:test'
             assert_file_contains ${package} 'finalize_nix_ci'
             assert_file_contains ${package} 'echo hello world'
-            assert_file_contains ${package} '"EXAMPLE":"empty"'
-            assert_file_contains ${package} '"SAMPLE":"working"'
-            assert_file_contains ${package} '"CURL":"/nix/store/.*-curl-.*"'
           '';
       }
       {
-        name = "simple pipeline deps drv";
+        name = "dont fail on store paths";
         type = "script";
         script = let
-          package = simplePipeline.packages."gitlab-ci:pipeline:test:job-deps:test";
+          package =
+            (cilib.mkCI {
+              pipelines."test" = {
+                variables.EXAMPLE = "${pkgs.hello}";
+              };
+            }).packages."gitlab-ci:pipeline:test";
         in
           # sh
           ''
             ${ntlib.helpers.path [pkgs.gnugrep]}
             ${ntlib.helpers.scriptHelpers}
-            assert_file_contains ${package} ':$PATH'
+            assert_file_contains ${package} '[".pre",".post"]'
-            assert_file_contains ${package} 'HELLO="/nix/store/.*-hello-.*"'
+            assert_file_contains ${package} '"EXAMPLE":"/nix/store/.*-hello-.*"'
           '';
       }
     ];