From 58a0db78613e772e0b82a265bb1fb6393f218f7e Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 11:22:55 +0100 Subject: [PATCH 1/6] tests(cilib_test) Extended test definition for the ci lib Extended the definition of tests for the continuous integration library, allowing for more in-depth testing of odd behavior of some library functions. --- tests/cilib_test.nix | 192 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 164 insertions(+), 28 deletions(-) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 78e99a3..02d1d29 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -56,24 +56,59 @@ } { name = "jobPatched nix disabled"; - expected = {}; + expected = { + /*cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + };*/ + }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = {}; + job = { + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + }; + }; nixConfig.enable = false; }; } { name = "jobPatched without runner cache"; expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + after_script = [ "finalize_nix_ci" ]; + before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; + variables = { + #BASH = toString pkgs.bash; + TEST = "work"; + }; + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = {}; + job = { + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + }; + }; nixConfig = { enable = true; enableRunnerCache = false; @@ -83,20 +118,34 @@ { name = "jobPatched with runner cache"; expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - cache = [ - { - key = "test"; - paths = [".nix-cache/"]; - } - ]; - variables."NIX_CI_CACHE_STRATEGY" = "runner"; + after_script = [ "finalize_nix_ci" ]; + before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; + cache = [{ + name = "some"; + paths = [ "this" ]; + }{ + key = "test"; + paths = [ ".nix-cache/" ]; + }]; + variables = { + NIX_CI_CACHE_STRATEGY = "runner"; + #BASH = toString pkgs.bash; + TEST = "work"; + }; }; actual = mkJobPatched { key = "test"; pipelineName = "test"; - job = {}; + job = { + cache = [{ + name = "some"; + paths = [ "this" ]; + }]; + variables = { + BASH = toString pkgs.bash; + TEST = "work"; + }; + }; nixConfig = { enable = true; enableRunnerCache = true; @@ -145,23 +194,110 @@ ''; } { - name = "handle store paths in variables"; + name = "handle store paths in variables without nix config"; expected = { - stages = ["test"]; - test.stage = "test"; + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + test = { + stage = "test"; + before_script = [ "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" ]; + /*cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + };*/ + }; }; - actual = - (mkPipeline { - name = "test"; - nixConfig.enable = false; - pipeline = { - stages = ["test"]; - jobs.test = { - stage = "test"; - variables."TEST" = "${pkgs.hello}"; + actual = (mkPipeline { + name = "test"; + nixConfig.enable = false; + pipeline = { + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + jobs.test = { + stage = "test"; + before_script = [ "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" ]; + cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; }; }; - }).finalConfig; + }; + }).finalConfig; + } + { + name = "handle store paths in variables with nix config"; + expected = { + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + test = { + stage = "test"; + before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" "finalize_nix_ci" ]; + cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }{ + key = "random"; + paths = [ ".nix-cache/" ]; + }]; + variables = { + SAMPLE = "working"; + #HELLO = toString pkgs.hello; + NIX_CI_CACHE_STRATEGY = "runner"; + }; + }; + }; + actual = (mkPipeline { + name = "test"; + nixConfig = { + enable = true; + enableRunnerCache = true; + runnerCacheKey = "random"; + }; + pipeline = { + stages = [ "test" ]; + variables = { + EXAMPLE = "empty"; + CURL = toString pkgs.curl; + }; + jobs.test = { + stage = "test"; + before_script = [ "./init" ]; + script = [ "echo Hello World!" ]; + after_script = [ "./clean" ]; + cache = [{ + key = "simple"; + paths = [ "~/random/" ]; + }]; + variables = { + SAMPLE = "working"; + HELLO = toString pkgs.hello; + }; + }; + }; + }).finalConfig; } ]; }; From f84edb77604b6af0143ef5816befd5e3c8de02f0 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 11:44:23 +0100 Subject: [PATCH 2/6] fix(storeDir) Use the builtin nix store location indicator Minor fixes in the variable filtering function with nix packages. --- lib/impl/helpers.nix | 2 +- tests/cilib_test.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/impl/helpers.nix b/lib/impl/helpers.nix index 48a79fc..754cead 100644 --- a/lib/impl/helpers.nix +++ b/lib/impl/helpers.nix @@ -50,7 +50,7 @@ in rec { filterJobVariables = shouldContain: job: concatMapAttrs ( name: value: - optionalAttrs ((hasInfix "/nix/store/" value) == shouldContain) { + optionalAttrs ((hasInfix builtins.storeDir value) == shouldContain) { ${name} = value; } ) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 02d1d29..fe041df 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -178,7 +178,7 @@ # sh '' set -euo pipefail - ${ntlib.helpers.path [pkgs.jq pkgs.gnugrep pkgs.coreutils]} + ${ntlib.helpers.path (with pkgs; [jq gnugrep coreutils])} echo "two keys, one json one pretty" jq 'keys | length == 2' "${pipeline}" | grep -q true echo "key[0] is exactly 'gitlab-ci:pipeline:test'" From d2f8a7067543246ce5219c478d855d17baa7e60c Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 12:33:55 +0100 Subject: [PATCH 3/6] fix(jobPatched) Removed too frequent filtering of variables Removed excessive disposal of environment variables containing paths to nix store package files, plus improved tests for this functionality. --- lib/impl/jobPatched.nix | 7 +++---- tests/cilib_test.nix | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix index c7134fb..4f2ddbd 100644 --- a/lib/impl/jobPatched.nix +++ b/lib/impl/jobPatched.nix @@ -3,7 +3,7 @@ helpers, }: let inherit (lib) toList optionalAttrs optional; - inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables; + inherit (helpers) prependToBeforeScript appendToAfterScript; in { key, @@ -18,9 +18,8 @@ in )) // optionalAttrs nixConfig.enable ( (let - variables = - (filterJobVariables false job) - // optionalAttrs nixConfig.enableRunnerCache { + variables = job.variables or {} // + optionalAttrs nixConfig.enableRunnerCache { NIX_CI_CACHE_STRATEGY = "runner"; }; in diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index fe041df..fea9dae 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -88,7 +88,7 @@ after_script = [ "finalize_nix_ci" ]; before_script = [ "source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\"" ]; variables = { - #BASH = toString pkgs.bash; + BASH = toString pkgs.bash; TEST = "work"; }; cache = [{ @@ -129,7 +129,7 @@ }]; variables = { NIX_CI_CACHE_STRATEGY = "runner"; - #BASH = toString pkgs.bash; + BASH = toString pkgs.bash; TEST = "work"; }; }; @@ -264,7 +264,7 @@ }]; variables = { SAMPLE = "working"; - #HELLO = toString pkgs.hello; + HELLO = toString pkgs.hello; NIX_CI_CACHE_STRATEGY = "runner"; }; }; From 0f9d0aae6043555ca6111a3476e5dcdc68f79117 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 12:55:22 +0100 Subject: [PATCH 4/6] fix(jobPatched) Fixed some attributes not being passed into ci Fixed modifying continuous integration settings attributes only when the job is configured via nix, plus improved tests for this functionality. --- lib/impl/jobPatched.nix | 54 +++++++++++++++++++---------------------- tests/cilib_test.nix | 8 +++--- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix index 4f2ddbd..73adb38 100644 --- a/lib/impl/jobPatched.nix +++ b/lib/impl/jobPatched.nix @@ -11,32 +11,28 @@ in pipelineName, nixConfig, }: - (builtins.removeAttrs job ["variables" "cache"]) - // (optionalAttrs nixConfig.enable ( - (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job) - // (appendToAfterScript ["finalize_nix_ci"] job) - )) - // optionalAttrs nixConfig.enable ( - (let - variables = job.variables or {} // - optionalAttrs nixConfig.enableRunnerCache { - NIX_CI_CACHE_STRATEGY = "runner"; - }; - in - # filter empty variables - optionalAttrs (variables != {}) { - inherit variables; - }) - // (let - cache = - (toList (job.cache or [])) - ++ (optional nixConfig.enableRunnerCache { - key = nixConfig.runnerCacheKey; - paths = [".nix-cache/"]; - }); - in - # filter empty cache - optionalAttrs (cache != []) { - inherit cache; - }) - ) + if ! nixConfig.enable then job else + (builtins.removeAttrs job [ "variables" "cache" ]) + // (prependToBeforeScript [ "source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\"" ] job) + // (appendToAfterScript [ "finalize_nix_ci" ] job) + // (let + variables = job.variables or {} // + optionalAttrs nixConfig.enableRunnerCache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + in + # filter empty variables + optionalAttrs (variables != {}) { + inherit variables; + }) + // (let + cache = (toList (job.cache or [])) ++ + (optional nixConfig.enableRunnerCache { + key = nixConfig.runnerCacheKey; + paths = [ ".nix-cache/" ]; + }); + in + # filter empty cache + optionalAttrs (cache != []) { + inherit cache; + }) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index fea9dae..4fc9933 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -57,14 +57,14 @@ { name = "jobPatched nix disabled"; expected = { - /*cache = [{ + cache = [{ name = "some"; paths = [ "this" ]; }]; variables = { BASH = toString pkgs.bash; TEST = "work"; - };*/ + }; }; actual = mkJobPatched { key = "test"; @@ -206,14 +206,14 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - /*cache = [{ + cache = [{ key = "simple"; paths = [ "~/random/" ]; }]; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; - };*/ + }; }; }; actual = (mkPipeline { From 8487c78246411312fa5459ac86da8670191bc7b7 Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 14:14:14 +0100 Subject: [PATCH 5/6] fix(ci) Fixed downloading images from project forks for ci Improved container image location guessing for repository forks. --- .gitlab-ci.yml | 2 ++ nix/repo/ci.nix | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bff742c..e8c146c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,3 +55,5 @@ stages: - build-images - build - trigger +variables: + NIX_CI_IMAGE: $CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA diff --git a/nix/repo/ci.nix b/nix/repo/ci.nix index a76c0d8..157ede7 100644 --- a/nix/repo/ci.nix +++ b/nix/repo/ci.nix @@ -10,6 +10,7 @@ in # the child pipeline can then use the built images to test them extraData = { stages = ["build-images" "build" "trigger"]; + variables.NIX_CI_IMAGE = "$CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA"; "build:image" = { stage = "build-images"; parallel.matrix = [ From fc79dd5120758733671f2618db216d9ed2a6d0fc Mon Sep 17 00:00:00 2001 From: Skryta Istota <6970043-hidden-being@users.noreply.gitlab.com> Date: Sun, 30 Nov 2025 14:44:44 +0100 Subject: [PATCH 6/6] chore(cilib_test) Improved tests for generating cache settings Changed tests for generating cache configurations to consider single settings for ci. --- tests/cilib_test.nix | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix index 4fc9933..6de4ed6 100644 --- a/tests/cilib_test.nix +++ b/tests/cilib_test.nix @@ -57,10 +57,10 @@ { name = "jobPatched nix disabled"; expected = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -70,10 +70,10 @@ key = "test"; pipelineName = "test"; job = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -100,10 +100,10 @@ key = "test"; pipelineName = "test"; job = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -137,10 +137,10 @@ key = "test"; pipelineName = "test"; job = { - cache = [{ + cache = { name = "some"; paths = [ "this" ]; - }]; + }; variables = { BASH = toString pkgs.bash; TEST = "work"; @@ -206,10 +206,10 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - cache = [{ + cache = { key = "simple"; paths = [ "~/random/" ]; - }]; + }; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; @@ -230,10 +230,10 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - cache = [{ + cache = { key = "simple"; paths = [ "~/random/" ]; - }]; + }; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello; @@ -287,10 +287,10 @@ before_script = [ "./init" ]; script = [ "echo Hello World!" ]; after_script = [ "./clean" ]; - cache = [{ + cache = { key = "simple"; paths = [ "~/random/" ]; - }]; + }; variables = { SAMPLE = "working"; HELLO = toString pkgs.hello;