diff --git a/.gitignore b/.gitignore index 2a8dde4..bbee8ad 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .direnv .pre-commit-config.yaml result +*.xml diff --git a/flake.lock b/flake.lock index 598eafa..56dfb71 100644 --- a/flake.lock +++ b/flake.lock @@ -332,11 +332,11 @@ "nixtest": { "locked": { "dir": "lib", - "lastModified": 1746388457, - "narHash": "sha256-xWAnSFxogdy47rZyGj18R9qDlLFs/PvwPpD3iAiR2Hc=", + "lastModified": 1748711200, + "narHash": "sha256-1Fx0jDk4ZsLkX3oSHo14OwzsK3sHkc2ykfz4bYqWuGA=", "owner": "technofab", "repo": "nixtest", - "rev": "0a1bbae2c30e3ba8e3b02de223e199c5dfd56572", + "rev": "3ff5b358d506473ace2311cd25fb95be0c048997", "type": "gitlab" }, "original": { diff --git a/flake.nix b/flake.nix index dfed39b..e4e363f 100644 --- a/flake.nix +++ b/flake.nix @@ -15,12 +15,16 @@ systems = import systems; flake = {}; perSystem = { + lib, pkgs, config, self', system, ... }: rec { + imports = [ + ./tests + ]; treefmt = { projectRootFile = "flake.nix"; programs = { @@ -201,7 +205,7 @@ stage = "nixtest"; script = [ # sh - "nix run .#nixtests:run -- --junit=junit.xml" + "nix run .#nixtests:run -- --junit=junit.xml --pure" ]; allow_failure = true; artifacts = { @@ -223,23 +227,6 @@ }; }; - nixtest.suites = let - jsonFile = file: builtins.fromJSON (builtins.readFile file); - in { - "Pipeline YAMLs" = [ - { - name = "default"; - type = "snapshot"; - actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:default"; - } - { - name = "non-default"; - type = "snapshot"; - actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:non-default"; - } - ]; - }; - packages = let setupScript = pkgs.writeShellScriptBin "setup_nix_ci" (builtins.readFile ./scripts/setup_nix_ci.sh); finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" (builtins.readFile ./scripts/finalize_nix_ci.sh); diff --git a/lib/default.nix b/lib/default.nix new file mode 100644 index 0000000..37f6318 --- /dev/null +++ b/lib/default.nix @@ -0,0 +1,8 @@ +args: { + helpers = import ./helpers.nix args; + jobDeps = import ./jobDeps.nix args; + jobRun = import ./jobRun.nix args; + jobPatch = import ./jobPatch.nix args; + pipeline = import ./pipeline.nix args; + utils = import ./utils.nix args; +} diff --git a/lib/flake.nix b/lib/flake.nix index 5d00bb6..b2cc77a 100644 --- a/lib/flake.nix +++ b/lib/flake.nix @@ -1,9 +1,8 @@ { - description = "Nix-CI lib"; + description = "Nix-GitLab-CI lib"; - outputs = {...} @ inputs: - { - flakeModule = import ./flakeModule.nix; - } - // (import ./utils.nix); + outputs = {...}: { + flakeModule = import ./flakeModule.nix; + lib = import ./default.nix; + }; } diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 3ba1e7e..4f8a274 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,22 +9,12 @@ pkgs, ... }: let - inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList; + cilib = import ./. {inherit lib pkgs;}; + inherit (cilib.pipeline) mkPipeline; + inherit (lib) types mkOption; + cfg = config.ci.config; - stdenvMinimal = pkgs.stdenvNoCC.override { - cc = null; - preHook = ""; - allowedRequisites = null; - initialPath = [pkgs.coreutils pkgs.findutils]; - extraNativeBuildInputs = []; - }; - - filterAttrsRec = pred: v: - if isAttrs v - then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) - else v; - subType = options: types.submodule {inherit options;}; mkNullOption = type: mkOption { @@ -168,228 +158,15 @@ }; }; - config.legacyPackages = let - # NOTE: json is also valid yaml and this removes dependency on jq - # and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml - # respectively) - toYaml = name: value: builtins.toFile name (builtins.toJSON value); - mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); - prepend = key: arr: job: - job - // lib.optionalAttrs job.nix.enable { - ${key} = - arr - ++ (job.${key} or []); - }; - append = key: arr: job: - job - // lib.optionalAttrs job.nix.enable { - ${key} = (job.${key} or []) ++ arr; - }; - prependToBeforeScript = prepend "before_script"; - appendToAfterScript = append "after_script"; - - # filter job's variables to either only those containing store paths - # or those that do not - filterJobVariables = nix: job: - lib.concatMapAttrs ( - name: value: - lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { - ${name} = value; - } - ) - (job.variables or {}); - in - lib.fold (pipeline: acc: acc // pipeline) {} (map ( - pipeline_name: let - pipeline = config.pipelines."${pipeline_name}"; - jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; - rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"; - value = stdenvMinimal.mkDerivation { - name = "gitlab-ci-job-deps-${key}"; - dontUnpack = true; - installPhase = let - script = '' - export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; - # variables containing nix derivations: - ${variableExports} - ''; - in - # sh - '' - echo '${script}' > $out - chmod +x $out - ''; - }; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - mapAttrs (key: job: let - variablesWithoutStorePaths = filterJobVariables false job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths - ); - in { - name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}"; - value = let - actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" '' - # set up deps and environment variables containing store paths - . ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - set -x - ${lib.concatLines (job.before_script or [])} - { set +x; } 2>/dev/null - echo -e "\e[32mRunning script...\e[0m" - set -x - ${lib.concatLines job.script} - { set +x; } 2>/dev/null - echo -e "\e[32mRunning after_script...\e[0m" - set -x - ${lib.concatLines (job.after_script or [])} - { set +x; } 2>/dev/null - ''; - sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" '' - echo -e "\e[32mSetting up...\e[0m" - - actualJobScript=$1 - shift - - INCLUDE_DIRTY=false - NO_SANDBOX=false - KEEP_TMP=false - KEEP_ENV="" - # parse flags - while [[ $# -gt 0 ]]; do - case "$1" in - --include-dirty) - INCLUDE_DIRTY=true - shift - ;; - --no-sandbox) - NO_SANDBOX=true - shift - ;; - --keep-tmp) - KEEP_TMP=true - shift - ;; - --keep-env) - KEEP_ENV="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac - done - - if [ "$NO_SANDBOX" = false ]; then - echo "Running with simple sandboxing" - if [ "$KEEP_TMP" = false ]; then - trap "rm -rf '$TMPDIR'" EXIT - else - echo "Temp dir will be preserved at: $TMPDIR" - fi - - # check if dirty - DIRTY_PATCH="" - if ! git diff --quiet && ! git diff --staged --quiet; then - echo "Warning: working tree is dirty." - DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch") - git diff --staged > "$DIRTY_PATCH" - trap "rm -f '$DIRTY_PATCH'" EXIT - fi - TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX") - git clone . $TMPDIR - pushd $TMPDIR >/dev/null - if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then - echo "Copying dirty changes..." - git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes" - fi - - echo "Running job in $TMPDIR" - env -i $( - if [[ -n "$KEEP_ENV" ]]; then - IFS=',' read -ra VARS <<< "$KEEP_ENV" - for var in "''${VARS[@]}"; do - printf '%s=%q ' "$var" "''${!var}" - done - fi - ) bash $actualJobScript - popd >/dev/null - else - exec $actualJobScript - fi - ''; - in - # this way the sandbox helper just needs to be built once - pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - exec ${lib.getExe sandboxHelper} ${actualJobScript} $@ - ''; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - mapAttrs (key: job: { - name = key; - value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - builtins.removeAttrs ( - (prependToBeforeScript [ - "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" - ] - (appendToAfterScript [ - "finalize_nix_ci" - ] - job)) - // lib.optionalAttrs job.nix.enable { - image = job.image; - variables = - (filterJobVariables false job) - // lib.optionalAttrs job.nix.enable-runner-cache { - NIX_CI_CACHE_STRATEGY = "runner"; - }; - cache = - ( - let - c = job.cache or []; - in - toList c - ) - ++ (lib.optional (job.nix.enable-runner-cache) { - key = job.nix.runner-cache-key; - paths = [".nix-cache/"]; - }); - } - ) ["nix"]; - }) - jobs; - in - # gitlab-ci:pipeline: - # gitlab-ci:pipeline::job: - # gitlab-ci:pipeline::job-deps: - { - "gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript - ) (builtins.attrNames config.pipelines)); + config.legacyPackages = lib.fold (pipeline: acc: acc // pipeline) {} ( + map ( + pipeline_name: + (mkPipeline { + pipeline = config.pipelines."${pipeline_name}"; + name = pipeline_name; + }).packages + ) (builtins.attrNames config.pipelines) + ); } ); } diff --git a/lib/helpers.nix b/lib/helpers.nix new file mode 100644 index 0000000..3b4c057 --- /dev/null +++ b/lib/helpers.nix @@ -0,0 +1,49 @@ +{lib, ...} @ args: let + inherit (lib) isAttrs filterAttrs mapAttrs; +in rec { + prepend = key: arr: job: + job + // lib.optionalAttrs (job.nix.enable or false) { + ${key} = + arr + ++ (job.${key} or []); + }; + append = key: arr: job: + job + // lib.optionalAttrs (job.nix.enable or false) { + ${key} = (job.${key} or []) ++ arr; + }; + prependToBeforeScript = prepend "before_script"; + appendToAfterScript = append "after_script"; + + # json is also valid yaml and this removes dependency on jq and/or remarshal + # (used in pkgs.formats.json and pkgs.formats.yaml respectively) + toYaml = name: value: builtins.toFile name (builtins.toJSON value); + + customMapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); + + filterAttrsRec = pred: v: + if isAttrs v + then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) + else v; + + # filter job's variables to either only those containing store paths + # or those that do not + filterJobVariables = nix: job: + lib.concatMapAttrs ( + name: value: + lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { + ${name} = value; + } + ) + (job.variables or {}); + + # args.pkgs so "pkgs" does not need to be passed all the time + stdenvMinimal = args.pkgs.stdenvNoCC.override { + cc = null; + preHook = ""; + allowedRequisites = null; + initialPath = with args.pkgs; [coreutils findutils]; + extraNativeBuildInputs = []; + }; +} diff --git a/lib/jobDeps.nix b/lib/jobDeps.nix new file mode 100644 index 0000000..27bad19 --- /dev/null +++ b/lib/jobDeps.nix @@ -0,0 +1,36 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./. {inherit lib pkgs;}; + inherit (cilib.helpers) filterJobVariables stdenvMinimal; +in { + mkJobDeps = { + key, + job, + }: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + script = '' + export PATH="${lib.makeBinPath (job.nix.deps or [])}:$PATH"; + # variables containing nix derivations: + ${variableExports} + ''; + in + stdenvMinimal.mkDerivation { + name = "gitlab-ci-job-deps-${key}"; + dontUnpack = true; + installPhase = + # sh + '' + echo '${script}' > $out + chmod +x $out + ''; + passthru = { + inherit script; + }; + }; +} diff --git a/lib/jobPatch.nix b/lib/jobPatch.nix new file mode 100644 index 0000000..959a9c0 --- /dev/null +++ b/lib/jobPatch.nix @@ -0,0 +1,49 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./. {inherit lib pkgs;}; + inherit (lib) toList; + inherit (cilib.helpers) prependToBeforeScript appendToAfterScript filterJobVariables; +in { + mkJobPatched = { + key, + job, + pipeline_name, + }: + builtins.removeAttrs ( + (prependToBeforeScript [ + "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" + ] + (appendToAfterScript [ + "finalize_nix_ci" + ] + job)) + // lib.optionalAttrs job.nix.enable ( + (let + variables = + (filterJobVariables false job) + // lib.optionalAttrs job.nix.enable-runner-cache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + in + # filter empty variables + lib.optionalAttrs (variables != {}) { + inherit variables; + }) + // (let + cache = + (toList (job.cache or [])) + ++ (lib.optional (job.nix.enable-runner-cache) { + key = job.nix.runner-cache-key; + paths = [".nix-cache/"]; + }); + in + # filter empty cache + lib.optionalAttrs (cache != []) { + inherit cache; + }) + ) + ) ["nix"]; +} diff --git a/lib/jobRun.nix b/lib/jobRun.nix new file mode 100644 index 0000000..5c4a118 --- /dev/null +++ b/lib/jobRun.nix @@ -0,0 +1,48 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./. {inherit lib pkgs;}; + inherit (cilib.helpers) filterJobVariables; +in { + mkJobRun = { + key, + job, + jobDeps, + }: let + variablesWithoutStorePaths = filterJobVariables false job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths + ); + sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" (builtins.readFile ./sandbox_helper.sh); + actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" '' + # set up deps and environment variables containing store paths + . ${jobDeps} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + set -x + ${lib.concatLines (job.before_script or [])} + { set +x; } 2>/dev/null + echo -e "\e[32mRunning script...\e[0m" + set -x + ${lib.concatLines job.script} + { set +x; } 2>/dev/null + echo -e "\e[32mRunning after_script...\e[0m" + set -x + ${lib.concatLines (job.after_script or [])} + { set +x; } 2>/dev/null + ''; + in + # this way the sandbox helper just needs to be built once + pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + exec ${lib.getExe sandboxHelper} ${actualJobScript} $@ + '' + // { + passthru = { + inherit jobDeps actualJobScript; + }; + }; +} diff --git a/lib/pipeline.nix b/lib/pipeline.nix new file mode 100644 index 0000000..2873b22 --- /dev/null +++ b/lib/pipeline.nix @@ -0,0 +1,61 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./. {inherit lib pkgs;}; + inherit (cilib.helpers) filterAttrsRec customMapAttrs toYaml; + inherit (cilib.jobDeps) mkJobDeps; + inherit (cilib.jobRun) mkJobRun; + inherit (cilib.jobPatch) mkJobPatched; +in { + mkPipeline = { + name, + pipeline, + }: let + jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; + rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + customMapAttrs (key: job: { + name = "gitlab-ci:pipeline:${name}:job-deps:${key}"; + value = mkJobDeps {inherit key job;}; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + customMapAttrs (key: job: { + name = "gitlab-ci:pipeline:${name}:job:${key}"; + value = mkJobRun { + inherit key job; + jobDeps = jobsMappedForDeps."gitlab-ci:pipeline:${name}:job-deps:${key}"; + }; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + customMapAttrs (key: job: { + name = key; + value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + mkJobPatched { + inherit key job; + pipeline_name = name; + }; + }) + jobs; + in { + packages = + # gitlab-ci:pipeline: + # gitlab-ci:pipeline::job: + # gitlab-ci:pipeline::job-deps: + { + "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-${name}.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript; + finalConfig = rest // jobsPatched; + }; +} diff --git a/lib/sandbox_helper.sh b/lib/sandbox_helper.sh new file mode 100644 index 0000000..df69d97 --- /dev/null +++ b/lib/sandbox_helper.sh @@ -0,0 +1,73 @@ +echo -e "\e[32mSetting up...\e[0m" + +actualJobScript=$1 +shift + +INCLUDE_DIRTY=false +NO_SANDBOX=false +KEEP_TMP=false +KEEP_ENV="" +# parse flags +while [[ $# -gt 0 ]]; do + case "$1" in + --include-dirty) + INCLUDE_DIRTY=true + shift + ;; + --no-sandbox) + NO_SANDBOX=true + shift + ;; + --keep-tmp) + KEEP_TMP=true + shift + ;; + --keep-env) + KEEP_ENV="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +if [ "$NO_SANDBOX" = false ]; then + echo "Running with simple sandboxing" + if [ "$KEEP_TMP" = false ]; then + trap "rm -rf '$TMPDIR'" EXIT + else + echo "Temp dir will be preserved at: $TMPDIR" + fi + + # check if dirty + DIRTY_PATCH="" + if ! git diff --quiet && ! git diff --staged --quiet; then + echo "Warning: working tree is dirty." + DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch") + git diff --staged > "$DIRTY_PATCH" + trap "rm -f '$DIRTY_PATCH'" EXIT + fi + TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX") + git clone . $TMPDIR + pushd $TMPDIR >/dev/null + if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then + echo "Copying dirty changes..." + git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes" + fi + + echo "Running job in $TMPDIR" + env -i $( + if [[ -n "$KEEP_ENV" ]]; then + IFS=',' read -ra VARS <<< "$KEEP_ENV" + for var in "''${VARS[@]}"; do + printf '%s=%q ' "$var" "''${!var}" + done + fi + ) bash $actualJobScript + popd >/dev/null +else + exec $actualJobScript +fi + diff --git a/lib/utils.nix b/lib/utils.nix index a40dadd..ae15284 100644 --- a/lib/utils.nix +++ b/lib/utils.nix @@ -1,46 +1,46 @@ -{ - mkUtils = {pkgs, ...}: { - commitAndPushFiles = { - message, - files ? [], - }: jobArgs: - jobArgs - // { - before_script = - (jobArgs.before_script or []) - ++ [ - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" - eval "$(ssh-agent -s)" >/dev/null; - mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; - ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; - echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; - git config --global user.email "$GIT_EMAIL" >/dev/null; - git config --global user.name "$GIT_NAME" >/dev/null; - export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; - git remote rm origin && git remote add origin ''${CI_PUSH_REPO} - echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" - '' - ]; - script = let - addScript = - if builtins.length files == 0 - then "" - else "git add ${builtins.concatStringsSep " " files}"; - in - (jobArgs.script or []) - ++ [ - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" - ${addScript} - git diff --cached --exit-code >/dev/null && - echo "Nothing to commit" || - git commit -m "${message}" --no-verify; - git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip - echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" - '' - ]; - nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; - }; - }; +{pkgs, ...}: { + commitAndPushFiles = { + message, + files ? [], + }: jobArgs: + jobArgs + // { + before_script = + (jobArgs.before_script or []) + ++ [ + # sh + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" + eval "$(ssh-agent -s)" >/dev/null; + mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; + ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; + echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; + git config --global user.email "$GIT_EMAIL" >/dev/null; + git config --global user.name "$GIT_NAME" >/dev/null; + export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; + git remote rm origin && git remote add origin ''${CI_PUSH_REPO} + echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" + '' + ]; + script = let + addScript = + if builtins.length files == 0 + then "" + else "git add ${builtins.concatStringsSep " " files}"; + in + (jobArgs.script or []) + ++ [ + # sh + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" + ${addScript} + git diff --cached --exit-code >/dev/null && + echo "Nothing to commit" || + git commit -m "${message}" --no-verify; + git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip + echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" + '' + ]; + nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; + }; } diff --git a/snapshots/default.snap.json b/snapshots/default.snap.json index 48ae054..1b9723c 100644 --- a/snapshots/default.snap.json +++ b/snapshots/default.snap.json @@ -1 +1 @@ -{"docs":{"after_script":["finalize_nix_ci"],"artifacts":{"paths":["public"]},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:docs\""],"cache":[],"image":"$NIX_CI_IMAGE","script":["nix build .#docs:default\nmkdir -p public\ncp -r result/. public/\n"],"stage":"build","variables":{}},"pages":{"artifacts":{"paths":["public"]},"image":"alpine:latest","rules":[{"if":"$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"}],"script":["true"],"stage":"deploy"},"stages":["test","build","deploy"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test\""],"cache":[{"key":"$CI_JOB_NAME-$CI_COMMIT_REF_SLUG","paths":[".nix-cache/"]}],"image":"$NIX_CI_IMAGE","script":["hello","curl google.de","echo $TEST $TEST_WITH_DERIVATION"],"stage":"test","variables":{"NIX_CI_CACHE_STRATEGY":"runner","TEST":"test"}},"test-default":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test-default\""],"cache":[],"image":"$NIX_CI_IMAGE","script":["hello"],"stage":"test","variables":{}},"test-non-nix":{"image":"alpine:latest","script":["echo \"This job will not be modified to use nix\""],"stage":"test"}} \ No newline at end of file +{"docs":{"after_script":["finalize_nix_ci"],"artifacts":{"paths":["public"]},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:docs\""],"image":"$NIX_CI_IMAGE","script":["nix build .#docs:default\nmkdir -p public\ncp -r result/. public/\n"],"stage":"build"},"nixtest":{"after_script":["finalize_nix_ci"],"allow_failure":true,"artifacts":{"reports":{"junit":"junit.xml"},"when":"always"},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:nixtest\""],"image":"$NIX_CI_IMAGE","script":["nix run .#nixtests:run -- --junit=junit.xml"],"stage":"nixtest"},"pages":{"artifacts":{"paths":["public"]},"image":"alpine:latest","rules":[{"if":"$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"}],"script":["true"],"stage":"deploy"},"stages":["test","nixtest","build","deploy"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test\""],"cache":[{"key":"$CI_JOB_NAME-$CI_COMMIT_REF_SLUG","paths":[".nix-cache/"]}],"image":"$NIX_CI_IMAGE","script":["hello","curl google.de","echo $TEST $TEST_WITH_DERIVATION"],"stage":"test","variables":{"NIX_CI_CACHE_STRATEGY":"runner","TEST":"test"}},"test-default":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test-default\""],"image":"$NIX_CI_IMAGE","script":["hello"],"stage":"test"},"test-non-nix":{"image":"alpine:latest","script":["echo \"This job will not be modified to use nix\""],"stage":"test"}} \ No newline at end of file diff --git a/snapshots/non-default.snap.json b/snapshots/non-default.snap.json index 6a613ef..9ed3256 100644 --- a/snapshots/non-default.snap.json +++ b/snapshots/non-default.snap.json @@ -1 +1 @@ -{"stages":["test"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:non-default:job-deps:test\""],"cache":[],"image":"$NIX_CI_IMAGE","script":["echo Hello from another pipeline"],"stage":"test","variables":{}}} \ No newline at end of file +{"stages":["test"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:non-default:job-deps:test\""],"image":"$NIX_CI_IMAGE","script":["echo Hello from another pipeline"],"stage":"test"}} \ No newline at end of file diff --git a/tests/ci-lib.nix b/tests/ci-lib.nix new file mode 100644 index 0000000..577139c --- /dev/null +++ b/tests/ci-lib.nix @@ -0,0 +1,140 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./../lib {inherit lib pkgs;}; +in { + nixtest.suites."CI Lib" = { + pos = __curPos; + tests = let + inherit (cilib.jobDeps) mkJobDeps; + inherit (cilib.jobRun) mkJobRun; + inherit (cilib.jobPatch) mkJobPatched; + inherit (cilib.pipeline) mkPipeline; + deps = mkJobDeps { + key = "test"; + job = { + nix.deps = [pkgs.hello]; + variables.TEST = "${pkgs.curl}"; + }; + }; + in [ + { + name = "jobDeps"; + type = "script"; + script = + # sh + '' + export PATH=${lib.makeBinPath [pkgs.gnugrep]} + grep -q "/nix/store" ${deps} + grep -q 'hello/bin:$PATH' ${deps} + grep -q "export TEST=" ${deps} + grep -q "curl" ${deps} + ''; + } + { + name = "jobRun"; + type = "script"; + script = let + run = mkJobRun { + key = "test"; + job.script = ["hello"]; + jobDeps = deps; + }; + in + # sh + '' + export PATH=${lib.makeBinPath [pkgs.gnugrep]} + grep -q "sandbox-helper" ${run}/bin/gitlab-ci-job:test + grep -q "gitlab-ci-job-test-raw" ${run}/bin/gitlab-ci-job:test + grep -q "gitlab-ci-job-deps-test" ${run.passthru.actualJobScript} + grep -q "Running script..." ${run.passthru.actualJobScript} + grep -q "hello" ${run.passthru.actualJobScript} + ''; + } + { + name = "jobPatched nix disabled"; + expected = {}; + actual = mkJobPatched { + key = "test"; + pipeline_name = "test"; + job.nix.enable = false; + }; + } + { + name = "jobPatched without runner cache"; + expected = { + after_script = ["finalize_nix_ci"]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + }; + actual = mkJobPatched { + key = "test"; + pipeline_name = "test"; + job.nix = { + enable = true; + enable-runner-cache = false; + }; + }; + } + { + name = "jobPatched with runner cache"; + expected = { + after_script = ["finalize_nix_ci"]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + cache = [ + { + key = "test"; + paths = [".nix-cache/"]; + } + ]; + variables."NIX_CI_CACHE_STRATEGY" = "runner"; + }; + actual = mkJobPatched { + key = "test"; + pipeline_name = "test"; + job.nix = { + enable = true; + enable-runner-cache = true; + runner-cache-key = "test"; + }; + }; + } + { + name = "mkPipeline empty"; + expected = {}; + actual = + (mkPipeline { + name = "test"; + pipeline.jobs = {}; + }).finalConfig; + } + { + name = "mkPipeline empty packages"; + type = "script"; + script = let + pipeline = builtins.toFile "pipeline-test" (builtins.toJSON + (mkPipeline { + name = "test"; + pipeline.jobs = {}; + }).packages); + in + # sh + '' + set -euo pipefail + export PATH=${lib.makeBinPath [pkgs.jq pkgs.gnugrep pkgs.coreutils]} + # single key + jq 'keys | length == 1' "${pipeline}" | grep -q true + # key is exactly "gitlab-ci:pipeline:test" + jq -r 'keys[0]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test" + # value contains "/nix/store/" + jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "/nix/store/" + # value contains "gitlab-ci-test.yml" + jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "gitlab-ci-test.yml" + # file only contains "{}" + [[ "$(cat $(jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}"))" == "{}" ]] + ''; + } + ]; + }; +} diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..764ae46 --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,8 @@ +{ + imports = [ + ./utils.nix + ./ci-lib.nix + ./helpers.nix + ./pipeline-yamls.nix + ]; +} diff --git a/tests/helpers.nix b/tests/helpers.nix new file mode 100644 index 0000000..1d100fc --- /dev/null +++ b/tests/helpers.nix @@ -0,0 +1,96 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./../lib {inherit lib pkgs;}; +in { + nixtest.suites."Helpers" = { + pos = __curPos; + tests = let + inherit (cilib) helpers; + in [ + { + name = "appendToAfterScript nix disabled"; + expected = {}; + actual = helpers.appendToAfterScript [] {}; + } + { + name = "appendToAfterScript empty"; + expected = { + nix.enable = true; + after_script = []; + }; + actual = helpers.appendToAfterScript [] {nix.enable = true;}; + } + { + name = "appendToAfterScript"; + expected = { + nix.enable = true; + after_script = ["echo after_script" "finalize_nix_ci"]; + }; + actual = helpers.appendToAfterScript ["finalize_nix_ci"] { + nix.enable = true; + after_script = ["echo after_script"]; + }; + } + { + name = "prependToBeforeScript nix disabled"; + expected = {}; + actual = helpers.prependToBeforeScript [] {}; + } + { + name = "prependToBeforeScript empty"; + expected = { + nix.enable = true; + before_script = []; + }; + actual = helpers.prependToBeforeScript [] {nix.enable = true;}; + } + { + name = "prependToBeforeScript"; + expected = { + nix.enable = true; + before_script = ["setup_nix_ci" "echo before_script"]; + }; + actual = helpers.prependToBeforeScript ["setup_nix_ci"] { + nix.enable = true; + before_script = ["echo before_script"]; + }; + } + { + name = "toYaml"; + expected = ''{"hello":"world"}''; + actual = builtins.readFile (helpers.toYaml "test" {hello = "world";}); + } + { + name = "filterAttrsRec"; + expected = {world = "world";}; + actual = helpers.filterAttrsRec (n: v: v != null) { + hello = null; + world = "world"; + }; + } + { + name = "filterJobVariables with store paths"; + expected = {HELLO = "${pkgs.hello}";}; + actual = helpers.filterJobVariables true { + variables = { + HELLO = "${pkgs.hello}"; + WORLD = "world"; + }; + }; + } + { + name = "filterJobVariables without store paths"; + expected = {WORLD = "world";}; + actual = helpers.filterJobVariables false { + variables = { + HELLO = "${pkgs.hello}"; + WORLD = "world"; + }; + }; + } + ]; + }; +} diff --git a/tests/pipeline-yamls.nix b/tests/pipeline-yamls.nix new file mode 100644 index 0000000..32effbe --- /dev/null +++ b/tests/pipeline-yamls.nix @@ -0,0 +1,26 @@ +{ + lib, + pkgs, + self', + ... +}: let + cilib = import ./../lib {inherit lib pkgs;}; +in { + nixtest.suites."Pipeline YAMLs" = { + pos = __curPos; + tests = let + jsonFile = file: builtins.fromJSON (builtins.readFile file); + in [ + { + name = "default"; + type = "snapshot"; + actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:default"; + } + { + name = "non-default"; + type = "snapshot"; + actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:non-default"; + } + ]; + }; +} diff --git a/tests/utils.nix b/tests/utils.nix new file mode 100644 index 0000000..8d9591f --- /dev/null +++ b/tests/utils.nix @@ -0,0 +1,36 @@ +{ + lib, + pkgs, + ... +}: let + cilib = import ./../lib {inherit lib pkgs;}; +in { + nixtest.suites."Utils" = { + pos = __curPos; + tests = [ + { + name = "commitAndPushFiles"; + type = "script"; + script = let + inherit (cilib) utils; + job = builtins.toFile "test" ( + builtins.unsafeDiscardStringContext ( + builtins.toJSON ( + utils.commitAndPushFiles { + message = "hello world"; + files = ["a.md" "b.txt"]; + } {} + ) + ) + ); + in + # sh + '' + export PATH=${lib.makeBinPath [pkgs.gnugrep]} + grep -q 'git commit -m \\"hello world\\"' ${job} + grep -q 'git add a.md b.txt' ${job} + ''; + } + ]; + }; +}