From 58a0db78613e772e0b82a265bb1fb6393f218f7e Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 11:22:55 +0100 Subject: [PATCH 01/12] tests(cilib_test) Extended test definition for the ci lib Extended the definition of tests for the continuous integration library, allowing for more in-depth testing of odd behavior of some library functions. --- tests/cilib_test.nix | 192 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 164 insertions(+), 28 deletions(-) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 78e99a3..02d1d29 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -56,24 +56,59 @@ } { name = "jobPatched nix disabled"; - expected = {}; + expected = { + /*cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + };*/ + }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = {}; + job = { + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + }; + }; nixConfig.enable = false; }; } { name = "jobPatched without runner cache"; expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + after_script = [ "finalize_nix_ci" ]; + before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; + variables = { + #BASH = toString pkgs.bash; + TEST = "work"; + }; + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = {}; + job = { + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + }; + }; nixConfig = { enable = true; enableRunnerCache = false; @@ -83,20 +118,34 @@ { name = "jobPatched with runner cache"; expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - cache = [ - { - key = "test"; - paths = [".nix-cache/"]; - } - ]; - variables."NIX_CI_CACHE_STRATEGY" = "runner"; + after_script = [ "finalize_nix_ci" ]; + before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; + cache = [{ + name = "some"; + paths = [ "this" ]; + }{ + key = "test"; + paths = [ ".nix-cache/" ]; + }]; + variables = { + NIX_CI_CACHE_STRATEGY = "runner"; + #BASH = toString pkgs.bash; + TEST = "work"; + }; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = {}; + job = { + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + }; + }; nixConfig = { enable = true; enableRunnerCache = true; @@ -145,23 +194,110 @@ ''; } { - name = "handle store paths in variables"; + name = "handle store paths in variables without nix config"; expected = { - stages = ["test"]; - test.stage = "test"; + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + test = { + stage = "test"; + before_script = [ "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" ]; + /*cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + };*/ + }; }; - actual = - (mkPipeline { - name = "test"; - nixConfig.enable = false; - pipeline = { - stages = ["test"]; - jobs.test = { - stage = "test"; - variables."TEST" = "${pkgs.hello}"; + actual = (mkPipeline { + name = "test"; + nixConfig.enable = false; + pipeline = { + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + jobs.test = { + stage = "test"; + before_script = [ "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" ]; + cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; }; }; - }).finalConfig; + }; + }).finalConfig; + } + { + name = "handle store paths in variables with nix config"; + expected = { + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + test = { + stage = "test"; + before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" "finalize_nix_ci" ]; + cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }{ + key = "random"; + paths = [ ".nix-cache/" ]; + }]; + variables = { + SAMPLE = "working"; + #HELLO = toString pkgs.hello; + NIX_CI_CACHE_STRATEGY = "runner"; + }; + }; + }; + actual = (mkPipeline { + name = "test"; + nixConfig = { + enable = true; + enableRunnerCache = true; + runnerCacheKey = "random"; + }; + pipeline = { + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + jobs.test = { + stage = "test"; + before_script = [ "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" ]; + cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + }; + }; + }; + }).finalConfig; } ]; }; From f84edb77604b6af0143ef5816befd5e3c8de02f0 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 11:44:23 +0100 Subject: [PATCH 02/12] fix(storeDir) Use the builtin nix store location indicator Minor fixes in the variable filtering function with nix packages. --- lib/impl/helpers.nix | 2 +- tests/cilib_test.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/impl/helpers.nix b/lib/impl/helpers.nix index 48a79fc..754cead 100644 --- a/lib/impl/helpers.nix +++ b/lib/impl/helpers.nix @@ -50,7 +50,7 @@ in rec { filterJobVariables = shouldContain: job: concatMapAttrs ( name: value: - optionalAttrs ((hasInfix "/nix/store/" value) == shouldContain) { + optionalAttrs ((hasInfix builtins.storeDir value) == shouldContain) { ${name} = value; } ) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 02d1d29..fe041df 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -178,7 +178,7 @@ # sh '' set -euo pipefail - ${ntlib.helpers.path [pkgs.jq pkgs.gnugrep pkgs.coreutils]} + ${ntlib.helpers.path (with pkgs; [jq gnugrep coreutils])} echo "two keys, one json one pretty" jq 'keys | length == 2' "${pipeline}" | grep -q true echo "key[0] is exactly 'gitlab-ci:pipeline:test'" From d2f8a7067543246ce5219c478d855d17baa7e60c Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 12:33:55 +0100 Subject: [PATCH 03/12] fix(jobPatched) Removed too frequent filtering of variables Removed excessive disposal of environment variables containing paths to nix store package files, plus improved tests for this functionality. --- lib/impl/jobPatched.nix | 7 +++---- tests/cilib_test.nix | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix index c7134fb..4f2ddbd 100644 --- a/lib/impl/jobPatched.nix +++ b/lib/impl/jobPatched.nix @@ -3,7 +3,7 @@ helpers, }: let inherit (lib) toList optionalAttrs optional; - inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables; + inherit (helpers) prependToBeforeScript appendToAfterScript; in { key, @@ -18,9 +18,8 @@ in )) // optionalAttrs nixConfig.enable ( (let - variables = - (filterJobVariables false job) - // optionalAttrs nixConfig.enableRunnerCache { + variables = job.variables or {} // + optionalAttrs nixConfig.enableRunnerCache { NIX_CI_CACHE_STRATEGY = "runner"; }; in diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index fe041df..fea9dae 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -88,7 +88,7 @@ after_script = [ "finalize_nix_ci" ]; before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; variables = { - #BASH = toString pkgs.bash; + BASH = toString pkgs.bash; TEST = "work"; }; cache = [{ @@ -129,7 +129,7 @@ }]; variables = { NIX_CI_CACHE_STRATEGY = "runner"; - #BASH = toString pkgs.bash; + BASH = toString pkgs.bash; TEST = "work"; }; }; @@ -264,7 +264,7 @@ }]; variables = { SAMPLE = "working"; - #HELLO = toString pkgs.hello; + HELLO = toString pkgs.hello; NIX_CI_CACHE_STRATEGY = "runner"; }; }; From 0f9d0aae6043555ca6111a3476e5dcdc68f79117 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 12:55:22 +0100 Subject: [PATCH 04/12] fix(jobPatched) Fixed some attributes not being passed into ci Fixed modifying continuous integration settings attributes only when the job is configured via nix, plus improved tests for this functionality. --- lib/impl/jobPatched.nix | 54 +++++++++++++++++++---------------------- tests/cilib_test.nix | 8 +++--- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix index 4f2ddbd..73adb38 100644 --- a/lib/impl/jobPatched.nix +++ b/lib/impl/jobPatched.nix @@ -11,32 +11,28 @@ in pipelineName, nixConfig, }: - (builtins.removeAttrs job ["variables" "cache"]) - // (optionalAttrs nixConfig.enable ( - (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job) - // (appendToAfterScript ["finalize_nix_ci"] job) - )) - // optionalAttrs nixConfig.enable ( - (let - variables = job.variables or {} // - optionalAttrs nixConfig.enableRunnerCache { - NIX_CI_CACHE_STRATEGY = "runner"; - }; - in - # filter empty variables - optionalAttrs (variables != {}) { - inherit variables; - }) - // (let - cache = - (toList (job.cache or [])) - ++ (optional nixConfig.enableRunnerCache { - key = nixConfig.runnerCacheKey; - paths = [".nix-cache/"]; - }); - in - # filter empty cache - optionalAttrs (cache != []) { - inherit cache; - }) - ) + if ! nixConfig.enable then job else + (builtins.removeAttrs job [ "variables" "cache" ]) + // (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\"" ] job) + // (appendToAfterScript [ "finalize_nix_ci" ] job) + // (let + variables = job.variables or {} // + optionalAttrs nixConfig.enableRunnerCache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + in + # filter empty variables + optionalAttrs (variables != {}) { + inherit variables; + }) + // (let + cache = (toList (job.cache or [])) ++ + (optional nixConfig.enableRunnerCache { + key = nixConfig.runnerCacheKey; + paths = [ ".nix-cache/" ]; + }); + in + # filter empty cache + optionalAttrs (cache != []) { + inherit cache; + }) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index fea9dae..4fc9933 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -57,14 +57,14 @@ { name = "jobPatched nix disabled"; expected = { - /*cache = [{ + cache = [{ name = "some"; paths = [ "this" ]; }]; variables = { BASH = toString pkgs.bash; TEST = "work"; - };*/ + }; }; actual = mkJobPatched { key = "test"; @@ -206,14 +206,14 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - /*cache = [{ + cache = [{ key = "simple"; paths = [ "~/random/" ]; }]; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; - };*/ + }; }; }; actual = (mkPipeline { From 8487c78246411312fa5459ac86da8670191bc7b7 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 14:14:14 +0100 Subject: [PATCH 05/12] fix(ci) Fixed downloading images from project forks for ci Improved container image location guessing for repository forks. --- .gitlab-ci.yml | 2 ++ nix/repo/ci.nix | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bff742c..e8c146c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,3 +55,5 @@ stages: - build-images - build - trigger +variables: + NIX_CI_IMAGE: $CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA diff --git a/nix/repo/ci.nix b/nix/repo/ci.nix index a76c0d8..157ede7 100644 --- a/nix/repo/ci.nix +++ b/nix/repo/ci.nix @@ -10,6 +10,7 @@ in # the child pipeline can then use the built images to test them extraData = { stages = ["build-images" "build" "trigger"]; + variables.NIX_CI_IMAGE = "$CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA"; "build:image" = { stage = "build-images"; parallel.matrix = [ From fc79dd5120758733671f2618db216d9ed2a6d0fc Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 14:44:44 +0100 Subject: [PATCH 06/12] chore(cilib_test) Improved tests for generating cache settings Changed tests for generating cache configurations to consider single settings for ci. --- tests/cilib_test.nix | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 4fc9933..6de4ed6 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -57,10 +57,10 @@ { name = "jobPatched nix disabled"; expected = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -70,10 +70,10 @@ key = "test"; pipelineName = "test"; job = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -100,10 +100,10 @@ key = "test"; pipelineName = "test"; job = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -137,10 +137,10 @@ key = "test"; pipelineName = "test"; job = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -206,10 +206,10 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - cache = [{ + cache = { key = "simple"; paths = [ "~/random/" ]; - }]; + }; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; @@ -230,10 +230,10 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - cache = [{ + cache = { key = "simple"; paths = [ "~/random/" ]; - }]; + }; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; @@ -287,10 +287,10 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - cache = [{ + cache = { key = "simple"; paths = [ "~/random/" ]; - }]; + }; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; From d8e55a05b2d5eaad335c5c038dc729c342a8e81f Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Tue, 2 Dec 2025 15:15:15 +0100 Subject: [PATCH 07/12] fix(nixfmt) Formatted modified files to default style The style of files modified in this branch has been corrected. --- lib/impl/jobPatched.nix | 54 +++++++------ tests/cilib_test.nix | 174 +++++++++++++++++++++------------------- 2 files changed, 121 insertions(+), 107 deletions(-) diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix index 73adb38..2467ae6 100644 --- a/lib/impl/jobPatched.nix +++ b/lib/impl/jobPatched.nix @@ -11,28 +11,32 @@ in pipelineName, nixConfig, }: - if ! nixConfig.enable then job else - (builtins.removeAttrs job [ "variables" "cache" ]) - // (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\"" ] job) - // (appendToAfterScript [ "finalize_nix_ci" ] job) - // (let - variables = job.variables or {} // - optionalAttrs nixConfig.enableRunnerCache { - NIX_CI_CACHE_STRATEGY = "runner"; - }; - in - # filter empty variables - optionalAttrs (variables != {}) { - inherit variables; - }) - // (let - cache = (toList (job.cache or [])) ++ - (optional nixConfig.enableRunnerCache { - key = nixConfig.runnerCacheKey; - paths = [ ".nix-cache/" ]; - }); - in - # filter empty cache - optionalAttrs (cache != []) { - inherit cache; - }) + if ! nixConfig.enable + then job + else + (builtins.removeAttrs job ["variables" "cache"]) + // (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job) + // (appendToAfterScript ["finalize_nix_ci"] job) + // (let + variables = + job.variables or {} + // optionalAttrs nixConfig.enableRunnerCache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + in + # filter empty variables + optionalAttrs (variables != {}) { + inherit variables; + }) + // (let + cache = + (toList (job.cache or [])) + ++ (optional nixConfig.enableRunnerCache { + key = nixConfig.runnerCacheKey; + paths = [".nix-cache/"]; + }); + in + # filter empty cache + optionalAttrs (cache != []) { + inherit cache; + }) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 6de4ed6..a46d24a 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -59,7 +59,7 @@ expected = { cache = { name = "some"; - paths = [ "this" ]; + paths = ["this"]; }; variables = { BASH = toString pkgs.bash; @@ -72,7 +72,7 @@ job = { cache = { name = "some"; - paths = [ "this" ]; + paths = ["this"]; }; variables = { BASH = toString pkgs.bash; @@ -85,16 +85,18 @@ { name = "jobPatched without runner cache"; expected = { - after_script = [ "finalize_nix_ci" ]; - before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; + after_script = ["finalize_nix_ci"]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; variables = { BASH = toString pkgs.bash; TEST = "work"; }; - cache = [{ - name = "some"; - paths = [ "this" ]; - }]; + cache = [ + { + name = "some"; + paths = ["this"]; + } + ]; }; actual = mkJobPatched { key = "test"; @@ -102,7 +104,7 @@ job = { cache = { name = "some"; - paths = [ "this" ]; + paths = ["this"]; }; variables = { BASH = toString pkgs.bash; @@ -118,15 +120,18 @@ { name = "jobPatched with runner cache"; expected = { - after_script = [ "finalize_nix_ci" ]; - before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; - cache = [{ - name = "some"; - paths = [ "this" ]; - }{ - key = "test"; - paths = [ ".nix-cache/" ]; - }]; + after_script = ["finalize_nix_ci"]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + cache = [ + { + name = "some"; + paths = ["this"]; + } + { + key = "test"; + paths = [".nix-cache/"]; + } + ]; variables = { NIX_CI_CACHE_STRATEGY = "runner"; BASH = toString pkgs.bash; @@ -139,7 +144,7 @@ job = { cache = { name = "some"; - paths = [ "this" ]; + paths = ["this"]; }; variables = { BASH = toString pkgs.bash; @@ -196,19 +201,19 @@ { name = "handle store paths in variables without nix config"; expected = { - stages = [ "test" ]; + stages = ["test"]; variables = { EXAMPLE = "empty"; CURL = toString pkgs.curl; }; test = { stage = "test"; - before_script = [ "./init" ]; - script = [ "echo Hello World!" ]; - after_script = [ "./clean" ]; + before_script = ["./init"]; + script = ["echo Hello World!"]; + after_script = ["./clean"]; cache = { key = "simple"; - paths = [ "~/random/" ]; + paths = ["~/random/"]; }; variables = { SAMPLE = "working"; @@ -216,52 +221,56 @@ }; }; }; - actual = (mkPipeline { - name = "test"; - nixConfig.enable = false; - pipeline = { - stages = [ "test" ]; - variables = { - EXAMPLE = "empty"; - CURL = toString pkgs.curl; - }; - jobs.test = { - stage = "test"; - before_script = [ "./init" ]; - script = [ "echo Hello World!" ]; - after_script = [ "./clean" ]; - cache = { - key = "simple"; - paths = [ "~/random/" ]; - }; + actual = + (mkPipeline { + name = "test"; + nixConfig.enable = false; + pipeline = { + stages = ["test"]; variables = { - SAMPLE = "working"; - HELLO = toString pkgs.hello; + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + jobs.test = { + stage = "test"; + before_script = ["./init"]; + script = ["echo Hello World!"]; + after_script = ["./clean"]; + cache = { + key = "simple"; + paths = ["~/random/"]; + }; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + }; }; }; - }; - }).finalConfig; + }).finalConfig; } { name = "handle store paths in variables with nix config"; expected = { - stages = [ "test" ]; + stages = ["test"]; variables = { EXAMPLE = "empty"; CURL = toString pkgs.curl; }; test = { stage = "test"; - before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init" ]; - script = [ "echo Hello World!" ]; - after_script = [ "./clean" "finalize_nix_ci" ]; - cache = [{ - key = "simple"; - paths = [ "~/random/" ]; - }{ - key = "random"; - paths = [ ".nix-cache/" ]; - }]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init"]; + script = ["echo Hello World!"]; + after_script = ["./clean" "finalize_nix_ci"]; + cache = [ + { + key = "simple"; + paths = ["~/random/"]; + } + { + key = "random"; + paths = [".nix-cache/"]; + } + ]; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; @@ -269,35 +278,36 @@ }; }; }; - actual = (mkPipeline { - name = "test"; - nixConfig = { - enable = true; - enableRunnerCache = true; - runnerCacheKey = "random"; - }; - pipeline = { - stages = [ "test" ]; - variables = { - EXAMPLE = "empty"; - CURL = toString pkgs.curl; + actual = + (mkPipeline { + name = "test"; + nixConfig = { + enable = true; + enableRunnerCache = true; + runnerCacheKey = "random"; }; - jobs.test = { - stage = "test"; - before_script = [ "./init" ]; - script = [ "echo Hello World!" ]; - after_script = [ "./clean" ]; - cache = { - key = "simple"; - paths = [ "~/random/" ]; - }; + pipeline = { + stages = ["test"]; variables = { - SAMPLE = "working"; - HELLO = toString pkgs.hello; + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + jobs.test = { + stage = "test"; + before_script = ["./init"]; + script = ["echo Hello World!"]; + after_script = ["./clean"]; + cache = { + key = "simple"; + paths = ["~/random/"]; + }; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + }; }; }; - }; - }).finalConfig; + }).finalConfig; } ]; }; From fb234b80bf11993c91b5d9a239d009f1f92c2d27 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Tue, 2 Dec 2025 15:35:56 +0100 Subject: [PATCH 08/12] chore(ci) Added job to check correct passing of variables Append a test for the correctness of transfer and creation of environment variables in ci, and added the required packages to run the remaining jobs in the terminal. --- nix/repo/ci.nix | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/nix/repo/ci.nix b/nix/repo/ci.nix index 157ede7..634305d 100644 --- a/nix/repo/ci.nix +++ b/nix/repo/ci.nix @@ -1,5 +1,5 @@ {inputs, ...}: let - inherit (inputs) cilib; + inherit (inputs) cilib pkgs; in cilib.mkCI { config.soonix = { @@ -58,9 +58,26 @@ in }; }; pipelines."default" = { - stages = ["test" "build" "deploy"]; + stages = ["check" "test" "build" "deploy"]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; jobs = { + "check" = { + stage = "check"; + script = [ + "set -euo pipefail" + "echo EXAMPLE=$EXAMPLE CURL=$CURL SAMPLE=$SAMPLE HELLO=$HELLO" + ]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + FF_SCRIPT_SECTIONS = "true"; + }; + }; "test" = { + nix.deps = with pkgs; [coreutils nix]; stage = "test"; script = [ "nix run .#tests -- --junit=junit.xml" @@ -73,6 +90,7 @@ in }; "docs" = { stage = "build"; + nix.deps = with pkgs; [coreutils nix]; script = [ # sh '' From 0ef6f4d2ff91ec41ef845ca506934c0492538e7b Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Wed, 3 Dec 2025 16:35:45 +0100 Subject: [PATCH 09/12] rollback(jobPatched) Restored some features for patching jobs for ci Tests for this filtering of environment variables in ci have also been rollback. --- lib/impl/jobPatched.nix | 4 ++-- nix/repo/ci.nix | 1 - tests/cilib_test.nix | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix index 2467ae6..367a374 100644 --- a/lib/impl/jobPatched.nix +++ b/lib/impl/jobPatched.nix @@ -3,7 +3,7 @@ helpers, }: let inherit (lib) toList optionalAttrs optional; - inherit (helpers) prependToBeforeScript appendToAfterScript; + inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables; in { key, @@ -19,7 +19,7 @@ in // (appendToAfterScript ["finalize_nix_ci"] job) // (let variables = - job.variables or {} + (filterJobVariables false job) // optionalAttrs nixConfig.enableRunnerCache { NIX_CI_CACHE_STRATEGY = "runner"; }; diff --git a/nix/repo/ci.nix b/nix/repo/ci.nix index 634305d..9ef9aac 100644 --- a/nix/repo/ci.nix +++ b/nix/repo/ci.nix @@ -73,7 +73,6 @@ in variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; - FF_SCRIPT_SECTIONS = "true"; }; }; "test" = { diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index a46d24a..1661c8c 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -88,7 +88,7 @@ after_script = ["finalize_nix_ci"]; before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; variables = { - BASH = toString pkgs.bash; + #BASH = toString pkgs.bash; TEST = "work"; }; cache = [ @@ -134,7 +134,7 @@ ]; variables = { NIX_CI_CACHE_STRATEGY = "runner"; - BASH = toString pkgs.bash; + #BASH = toString pkgs.bash; TEST = "work"; }; }; @@ -273,7 +273,7 @@ ]; variables = { SAMPLE = "working"; - HELLO = toString pkgs.hello; + #HELLO = toString pkgs.hello; NIX_CI_CACHE_STRATEGY = "runner"; }; }; From 8336e6d0839fa4b22ad0a2035da5280bfb97a0ce Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Wed, 3 Dec 2025 16:55:05 +0100 Subject: [PATCH 10/12] tests(modules_test) Extended test definition for modules Extended the definition of tests for the continuous integration modules, allowing for more in-depth testing of odd behavior of some configuration generators. --- tests/modules_test.nix | 47 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/tests/modules_test.nix b/tests/modules_test.nix index e134293..e56ad34 100644 --- a/tests/modules_test.nix +++ b/tests/modules_test.nix @@ -10,9 +10,17 @@ simplePipeline = cilib.mkCI { pipelines."test" = { stages = ["test"]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; jobs."test" = { stage = "test"; script = ["echo hello world"]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + }; }; }; }; @@ -30,21 +38,29 @@ (cilib.mkCI {}).packages; } { - name = "simple pipeline"; + name = "simple pipeline final config"; expected = { stages = [".pre" "test" ".post"]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; "test" = { image = "$NIX_CI_IMAGE"; stage = "test"; before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; script = ["echo hello world"]; after_script = ["finalize_nix_ci"]; + variables = { + SAMPLE = "working"; + #HELLO = toString pkgs.hello; + }; }; }; actual = simplePipeline.pipelines."test".finalConfig; } { - name = "simple pipeline yaml"; + name = "simple pipeline json"; type = "script"; script = let package = simplePipeline.packages."gitlab-ci:pipeline:test"; @@ -56,16 +72,36 @@ assert_file_contains ${package} 'gitlab-ci:pipeline:test:job-deps:test' assert_file_contains ${package} 'finalize_nix_ci' assert_file_contains ${package} 'echo hello world' + assert_file_contains ${package} '"EXAMPLE":"empty"' + assert_file_contains ${package} '"SAMPLE":"working"' + assert_file_contains ${package} '"CURL":"/nix/store/.*-curl-.*"' ''; } { - name = "dont fail on store paths"; + name = "simple pipeline deps drv"; + type = "script"; + script = let + package = simplePipeline.packages."gitlab-ci:pipeline:test:job-deps:test"; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${package} ':$PATH' + assert_file_contains ${package} 'HELLO="/nix/store/.*-hello-.*"' + ''; + } + { + name = "do not fail on store paths"; type = "script"; script = let package = (cilib.mkCI { pipelines."test" = { - variables.EXAMPLE = "${pkgs.hello}"; + variables = { + HELLO = "${pkgs.hello}"; + SAMPLE = "empty"; + }; }; }).packages."gitlab-ci:pipeline:test"; in @@ -74,7 +110,8 @@ ${ntlib.helpers.path [pkgs.gnugrep]} ${ntlib.helpers.scriptHelpers} assert_file_contains ${package} '[".pre",".post"]' - assert_file_contains ${package} '"EXAMPLE":"/nix/store/.*-hello-.*"' + assert_file_contains ${package} '"HELLO":"/nix/store/.*-hello-.*"' + assert_file_contains ${package} '"SAMPLE":"empty"' ''; } ]; From c8f51c73eda5f0d2bcdd7c8f80769c67de94534d Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Wed, 3 Dec 2025 17:35:36 +0100 Subject: [PATCH 11/12] fix(job) Fixed premature filtering of variables Fixed excessive deletion of environment variables containing paths to nix store package files, causing them to not be exported in Giltab's CI environment, plus removed duplicate tests for moving environment variables. --- lib/impl/modules/job.nix | 2 +- tests/modules_test.nix | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/lib/impl/modules/job.nix b/lib/impl/modules/job.nix index 295093d..1f99d5b 100644 --- a/lib/impl/modules/job.nix +++ b/lib/impl/modules/job.nix @@ -646,7 +646,7 @@ in rec { }; depsDrv = cilib.mkJobDeps { key = name; - job = config.finalConfig; + inherit job; nixConfig = config.nix; }; runnerDrv = cilib.mkJobRun { diff --git a/tests/modules_test.nix b/tests/modules_test.nix index e56ad34..0bd19c6 100644 --- a/tests/modules_test.nix +++ b/tests/modules_test.nix @@ -91,29 +91,6 @@ assert_file_contains ${package} 'HELLO="/nix/store/.*-hello-.*"' ''; } - { - name = "do not fail on store paths"; - type = "script"; - script = let - package = - (cilib.mkCI { - pipelines."test" = { - variables = { - HELLO = "${pkgs.hello}"; - SAMPLE = "empty"; - }; - }; - }).packages."gitlab-ci:pipeline:test"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${package} '[".pre",".post"]' - assert_file_contains ${package} '"HELLO":"/nix/store/.*-hello-.*"' - assert_file_contains ${package} '"SAMPLE":"empty"' - ''; - } ]; }; } From 65b90ecd653c8002a2a53792f5a9bafe8b8d857b Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Wed, 3 Dec 2025 18:35:36 +0100 Subject: [PATCH 12/12] rollback(cilib_test) Synchronized the test file of the Ci library Updated the continuous integration test file to be compatible with the newer main branch. --- tests/cilib_test.nix | 157 +++++++------------------------------------ 1 file changed, 26 insertions(+), 131 deletions(-) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 1661c8c..3b76fb8 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -56,28 +56,26 @@ } { name = "jobPatched nix disabled"; + expected = {}; + actual = mkJobPatched { + key = "test"; + pipelineName = "test"; + job = {}; + nixConfig.enable = false; + }; + } + { + name = "jobPatched nix disabled with variables and cache"; expected = { - cache = { - name = "some"; - paths = ["this"]; - }; - variables = { - BASH = toString pkgs.bash; - TEST = "work"; - }; + variables."HELLO" = "world"; + cache = [{key = "example";}]; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; job = { - cache = { - name = "some"; - paths = ["this"]; - }; - variables = { - BASH = toString pkgs.bash; - TEST = "work"; - }; + variables."HELLO" = "world"; + cache = [{key = "example";}]; }; nixConfig.enable = false; }; @@ -87,30 +85,11 @@ expected = { after_script = ["finalize_nix_ci"]; before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - variables = { - #BASH = toString pkgs.bash; - TEST = "work"; - }; - cache = [ - { - name = "some"; - paths = ["this"]; - } - ]; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = { - cache = { - name = "some"; - paths = ["this"]; - }; - variables = { - BASH = toString pkgs.bash; - TEST = "work"; - }; - }; + job = {}; nixConfig = { enable = true; enableRunnerCache = false; @@ -123,34 +102,17 @@ after_script = ["finalize_nix_ci"]; before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; cache = [ - { - name = "some"; - paths = ["this"]; - } { key = "test"; paths = [".nix-cache/"]; } ]; - variables = { - NIX_CI_CACHE_STRATEGY = "runner"; - #BASH = toString pkgs.bash; - TEST = "work"; - }; + variables."NIX_CI_CACHE_STRATEGY" = "runner"; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = { - cache = { - name = "some"; - paths = ["this"]; - }; - variables = { - BASH = toString pkgs.bash; - TEST = "work"; - }; - }; + job = {}; nixConfig = { enable = true; enableRunnerCache = true; @@ -199,26 +161,12 @@ ''; } { - name = "handle store paths in variables without nix config"; + name = "ignore store paths in variables with nix disabled"; expected = { stages = ["test"]; - variables = { - EXAMPLE = "empty"; - CURL = toString pkgs.curl; - }; test = { stage = "test"; - before_script = ["./init"]; - script = ["echo Hello World!"]; - after_script = ["./clean"]; - cache = { - key = "simple"; - paths = ["~/random/"]; - }; - variables = { - SAMPLE = "working"; - HELLO = toString pkgs.hello; - }; + variables."TEST" = "${pkgs.hello}"; }; }; actual = @@ -227,85 +175,32 @@ nixConfig.enable = false; pipeline = { stages = ["test"]; - variables = { - EXAMPLE = "empty"; - CURL = toString pkgs.curl; - }; jobs.test = { stage = "test"; - before_script = ["./init"]; - script = ["echo Hello World!"]; - after_script = ["./clean"]; - cache = { - key = "simple"; - paths = ["~/random/"]; - }; - variables = { - SAMPLE = "working"; - HELLO = toString pkgs.hello; - }; + variables."TEST" = "${pkgs.hello}"; }; }; }).finalConfig; } { - name = "handle store paths in variables with nix config"; + # it doesn't make much sense to have any nix store path in variables, but we ignore it for global variables + name = "ignore store paths in global variables"; expected = { - stages = ["test"]; variables = { - EXAMPLE = "empty"; + HELLO = "world"; CURL = toString pkgs.curl; }; - test = { - stage = "test"; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init"]; - script = ["echo Hello World!"]; - after_script = ["./clean" "finalize_nix_ci"]; - cache = [ - { - key = "simple"; - paths = ["~/random/"]; - } - { - key = "random"; - paths = [".nix-cache/"]; - } - ]; - variables = { - SAMPLE = "working"; - #HELLO = toString pkgs.hello; - NIX_CI_CACHE_STRATEGY = "runner"; - }; - }; }; actual = (mkPipeline { name = "test"; - nixConfig = { - enable = true; - enableRunnerCache = true; - runnerCacheKey = "random"; - }; + nixConfig.enable = true; pipeline = { - stages = ["test"]; variables = { - EXAMPLE = "empty"; + HELLO = "world"; CURL = toString pkgs.curl; }; - jobs.test = { - stage = "test"; - before_script = ["./init"]; - script = ["echo Hello World!"]; - after_script = ["./clean"]; - cache = { - key = "simple"; - paths = ["~/random/"]; - }; - variables = { - SAMPLE = "working"; - HELLO = toString pkgs.hello; - }; - }; + jobs = {}; }; }).finalConfig; }