diff --git a/.gitignore b/.gitignore index bbee8ad..2a8dde4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,3 @@ .direnv .pre-commit-config.yaml result -*.xml diff --git a/flake.lock b/flake.lock index 38bc581..f1a0875 100644 --- a/flake.lock +++ b/flake.lock @@ -329,24 +329,6 @@ "type": "github" } }, - "nixtest": { - "locked": { - "dir": "lib", - "lastModified": 1748945995, - "narHash": "sha256-OWflapMq78nCUT7N0vS/AuK4E8y7p8oh0zjnDioJ4Lw=", - "owner": "technofab", - "repo": "nixtest", - "rev": "392e8796f386373a36aecd3216925accf2714a1a", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "technofab", - "ref": "1.0.0", - "repo": "nixtest", - "type": "gitlab" - } - }, "root": { "inputs": { "devenv": "devenv", @@ -354,7 +336,6 @@ "mkdocs-material-umami": "mkdocs-material-umami", "nix-mkdocs": "nix-mkdocs", "nixpkgs": "nixpkgs_4", - "nixtest": "nixtest", "systems": "systems", "treefmt-nix": "treefmt-nix" } diff --git a/flake.nix b/flake.nix index 0788dc0..6753d41 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,6 @@ inputs.devenv.flakeModule inputs.treefmt-nix.flakeModule inputs.nix-mkdocs.flakeModule - inputs.nixtest.flakeModule ./lib/flakeModule.nix ]; systems = import systems; @@ -20,9 +19,6 @@ system, ... }: rec { - imports = [ - ./tests - ]; treefmt = { projectRootFile = "flake.nix"; programs = { @@ -143,7 +139,7 @@ }; # should set the "default" pipeline ci = { - stages = ["test" "nixtest" "build" "deploy"]; + stages = ["test" "build" "deploy"]; jobs = { "test" = { stage = "test"; @@ -199,18 +195,6 @@ } ]; }; - "nixtest" = { - stage = "nixtest"; - script = [ - # sh - "nix run .#nixtests:run -- --junit=junit.xml --pure" - ]; - allow_failure = true; - artifacts = { - when = "always"; - reports.junit = "junit.xml"; - }; - }; }; }; pipelines."non-default" = { @@ -285,7 +269,6 @@ treefmt-nix.url = "github:numtide/treefmt-nix"; nix-mkdocs.url = "gitlab:technofab/nixmkdocs?dir=lib"; mkdocs-material-umami.url = "gitlab:technofab/mkdocs-material-umami"; - nixtest.url = "gitlab:technofab/nixtest/1.0.0?dir=lib"; }; nixConfig = { diff --git a/lib/default.nix b/lib/default.nix deleted file mode 100644 index 37f6318..0000000 --- a/lib/default.nix +++ /dev/null @@ -1,8 +0,0 @@ -args: { - helpers = import ./helpers.nix args; - jobDeps = import ./jobDeps.nix args; - jobRun = import ./jobRun.nix args; - jobPatch = import ./jobPatch.nix args; - pipeline = import ./pipeline.nix args; - utils = import ./utils.nix args; -} diff --git a/lib/flake.nix b/lib/flake.nix index b2cc77a..5d00bb6 100644 --- a/lib/flake.nix +++ b/lib/flake.nix @@ -1,8 +1,9 @@ { - description = "Nix-GitLab-CI lib"; + description = "Nix-CI lib"; - outputs = {...}: { - flakeModule = import ./flakeModule.nix; - lib = import ./default.nix; - }; + outputs = {...} @ inputs: + { + flakeModule = import ./flakeModule.nix; + } + // (import ./utils.nix); } diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 4f8a274..3ba1e7e 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,12 +9,22 @@ pkgs, ... }: let - cilib = import ./. {inherit lib pkgs;}; - inherit (cilib.pipeline) mkPipeline; - inherit (lib) types mkOption; - + inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList; cfg = config.ci.config; + stdenvMinimal = pkgs.stdenvNoCC.override { + cc = null; + preHook = ""; + allowedRequisites = null; + initialPath = [pkgs.coreutils pkgs.findutils]; + extraNativeBuildInputs = []; + }; + + filterAttrsRec = pred: v: + if isAttrs v + then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) + else v; + subType = options: types.submodule {inherit options;}; mkNullOption = type: mkOption { @@ -158,15 +168,228 @@ }; }; - config.legacyPackages = lib.fold (pipeline: acc: acc // pipeline) {} ( - map ( - pipeline_name: - (mkPipeline { - pipeline = config.pipelines."${pipeline_name}"; - name = pipeline_name; - }).packages - ) (builtins.attrNames config.pipelines) - ); + config.legacyPackages = let + # NOTE: json is also valid yaml and this removes dependency on jq + # and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml + # respectively) + toYaml = name: value: builtins.toFile name (builtins.toJSON value); + mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); + prepend = key: arr: job: + job + // lib.optionalAttrs job.nix.enable { + ${key} = + arr + ++ (job.${key} or []); + }; + append = key: arr: job: + job + // lib.optionalAttrs job.nix.enable { + ${key} = (job.${key} or []) ++ arr; + }; + prependToBeforeScript = prepend "before_script"; + appendToAfterScript = append "after_script"; + + # filter job's variables to either only those containing store paths + # or those that do not + filterJobVariables = nix: job: + lib.concatMapAttrs ( + name: value: + lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { + ${name} = value; + } + ) + (job.variables or {}); + in + lib.fold (pipeline: acc: acc // pipeline) {} (map ( + pipeline_name: let + pipeline = config.pipelines."${pipeline_name}"; + jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; + rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + mapAttrs (key: job: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"; + value = stdenvMinimal.mkDerivation { + name = "gitlab-ci-job-deps-${key}"; + dontUnpack = true; + installPhase = let + script = '' + export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; + # variables containing nix derivations: + ${variableExports} + ''; + in + # sh + '' + echo '${script}' > $out + chmod +x $out + ''; + }; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + mapAttrs (key: job: let + variablesWithoutStorePaths = filterJobVariables false job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}"; + value = let + actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" '' + # set up deps and environment variables containing store paths + . ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + set -x + ${lib.concatLines (job.before_script or [])} + { set +x; } 2>/dev/null + echo -e "\e[32mRunning script...\e[0m" + set -x + ${lib.concatLines job.script} + { set +x; } 2>/dev/null + echo -e "\e[32mRunning after_script...\e[0m" + set -x + ${lib.concatLines (job.after_script or [])} + { set +x; } 2>/dev/null + ''; + sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" '' + echo -e "\e[32mSetting up...\e[0m" + + actualJobScript=$1 + shift + + INCLUDE_DIRTY=false + NO_SANDBOX=false + KEEP_TMP=false + KEEP_ENV="" + # parse flags + while [[ $# -gt 0 ]]; do + case "$1" in + --include-dirty) + INCLUDE_DIRTY=true + shift + ;; + --no-sandbox) + NO_SANDBOX=true + shift + ;; + --keep-tmp) + KEEP_TMP=true + shift + ;; + --keep-env) + KEEP_ENV="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac + done + + if [ "$NO_SANDBOX" = false ]; then + echo "Running with simple sandboxing" + if [ "$KEEP_TMP" = false ]; then + trap "rm -rf '$TMPDIR'" EXIT + else + echo "Temp dir will be preserved at: $TMPDIR" + fi + + # check if dirty + DIRTY_PATCH="" + if ! git diff --quiet && ! git diff --staged --quiet; then + echo "Warning: working tree is dirty." + DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch") + git diff --staged > "$DIRTY_PATCH" + trap "rm -f '$DIRTY_PATCH'" EXIT + fi + TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX") + git clone . $TMPDIR + pushd $TMPDIR >/dev/null + if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then + echo "Copying dirty changes..." + git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes" + fi + + echo "Running job in $TMPDIR" + env -i $( + if [[ -n "$KEEP_ENV" ]]; then + IFS=',' read -ra VARS <<< "$KEEP_ENV" + for var in "''${VARS[@]}"; do + printf '%s=%q ' "$var" "''${!var}" + done + fi + ) bash $actualJobScript + popd >/dev/null + else + exec $actualJobScript + fi + ''; + in + # this way the sandbox helper just needs to be built once + pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + exec ${lib.getExe sandboxHelper} ${actualJobScript} $@ + ''; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + mapAttrs (key: job: { + name = key; + value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + builtins.removeAttrs ( + (prependToBeforeScript [ + "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" + ] + (appendToAfterScript [ + "finalize_nix_ci" + ] + job)) + // lib.optionalAttrs job.nix.enable { + image = job.image; + variables = + (filterJobVariables false job) + // lib.optionalAttrs job.nix.enable-runner-cache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + cache = + ( + let + c = job.cache or []; + in + toList c + ) + ++ (lib.optional (job.nix.enable-runner-cache) { + key = job.nix.runner-cache-key; + paths = [".nix-cache/"]; + }); + } + ) ["nix"]; + }) + jobs; + in + # gitlab-ci:pipeline: + # gitlab-ci:pipeline::job: + # gitlab-ci:pipeline::job-deps: + { + "gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript + ) (builtins.attrNames config.pipelines)); } ); } diff --git a/lib/helpers.nix b/lib/helpers.nix deleted file mode 100644 index 3b4c057..0000000 --- a/lib/helpers.nix +++ /dev/null @@ -1,49 +0,0 @@ -{lib, ...} @ args: let - inherit (lib) isAttrs filterAttrs mapAttrs; -in rec { - prepend = key: arr: job: - job - // lib.optionalAttrs (job.nix.enable or false) { - ${key} = - arr - ++ (job.${key} or []); - }; - append = key: arr: job: - job - // lib.optionalAttrs (job.nix.enable or false) { - ${key} = (job.${key} or []) ++ arr; - }; - prependToBeforeScript = prepend "before_script"; - appendToAfterScript = append "after_script"; - - # json is also valid yaml and this removes dependency on jq and/or remarshal - # (used in pkgs.formats.json and pkgs.formats.yaml respectively) - toYaml = name: value: builtins.toFile name (builtins.toJSON value); - - customMapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); - - filterAttrsRec = pred: v: - if isAttrs v - then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) - else v; - - # filter job's variables to either only those containing store paths - # or those that do not - filterJobVariables = nix: job: - lib.concatMapAttrs ( - name: value: - lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { - ${name} = value; - } - ) - (job.variables or {}); - - # args.pkgs so "pkgs" does not need to be passed all the time - stdenvMinimal = args.pkgs.stdenvNoCC.override { - cc = null; - preHook = ""; - allowedRequisites = null; - initialPath = with args.pkgs; [coreutils findutils]; - extraNativeBuildInputs = []; - }; -} diff --git a/lib/jobDeps.nix b/lib/jobDeps.nix deleted file mode 100644 index 27bad19..0000000 --- a/lib/jobDeps.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./. {inherit lib pkgs;}; - inherit (cilib.helpers) filterJobVariables stdenvMinimal; -in { - mkJobDeps = { - key, - job, - }: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - script = '' - export PATH="${lib.makeBinPath (job.nix.deps or [])}:$PATH"; - # variables containing nix derivations: - ${variableExports} - ''; - in - stdenvMinimal.mkDerivation { - name = "gitlab-ci-job-deps-${key}"; - dontUnpack = true; - installPhase = - # sh - '' - echo '${script}' > $out - chmod +x $out - ''; - passthru = { - inherit script; - }; - }; -} diff --git a/lib/jobPatch.nix b/lib/jobPatch.nix deleted file mode 100644 index 959a9c0..0000000 --- a/lib/jobPatch.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./. {inherit lib pkgs;}; - inherit (lib) toList; - inherit (cilib.helpers) prependToBeforeScript appendToAfterScript filterJobVariables; -in { - mkJobPatched = { - key, - job, - pipeline_name, - }: - builtins.removeAttrs ( - (prependToBeforeScript [ - "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" - ] - (appendToAfterScript [ - "finalize_nix_ci" - ] - job)) - // lib.optionalAttrs job.nix.enable ( - (let - variables = - (filterJobVariables false job) - // lib.optionalAttrs job.nix.enable-runner-cache { - NIX_CI_CACHE_STRATEGY = "runner"; - }; - in - # filter empty variables - lib.optionalAttrs (variables != {}) { - inherit variables; - }) - // (let - cache = - (toList (job.cache or [])) - ++ (lib.optional (job.nix.enable-runner-cache) { - key = job.nix.runner-cache-key; - paths = [".nix-cache/"]; - }); - in - # filter empty cache - lib.optionalAttrs (cache != []) { - inherit cache; - }) - ) - ) ["nix"]; -} diff --git a/lib/jobRun.nix b/lib/jobRun.nix deleted file mode 100644 index 5c4a118..0000000 --- a/lib/jobRun.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./. {inherit lib pkgs;}; - inherit (cilib.helpers) filterJobVariables; -in { - mkJobRun = { - key, - job, - jobDeps, - }: let - variablesWithoutStorePaths = filterJobVariables false job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths - ); - sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" (builtins.readFile ./sandbox_helper.sh); - actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" '' - # set up deps and environment variables containing store paths - . ${jobDeps} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - set -x - ${lib.concatLines (job.before_script or [])} - { set +x; } 2>/dev/null - echo -e "\e[32mRunning script...\e[0m" - set -x - ${lib.concatLines job.script} - { set +x; } 2>/dev/null - echo -e "\e[32mRunning after_script...\e[0m" - set -x - ${lib.concatLines (job.after_script or [])} - { set +x; } 2>/dev/null - ''; - in - # this way the sandbox helper just needs to be built once - pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - exec ${lib.getExe sandboxHelper} ${actualJobScript} $@ - '' - // { - passthru = { - inherit jobDeps actualJobScript; - }; - }; -} diff --git a/lib/pipeline.nix b/lib/pipeline.nix deleted file mode 100644 index 2873b22..0000000 --- a/lib/pipeline.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./. {inherit lib pkgs;}; - inherit (cilib.helpers) filterAttrsRec customMapAttrs toYaml; - inherit (cilib.jobDeps) mkJobDeps; - inherit (cilib.jobRun) mkJobRun; - inherit (cilib.jobPatch) mkJobPatched; -in { - mkPipeline = { - name, - pipeline, - }: let - jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; - rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - customMapAttrs (key: job: { - name = "gitlab-ci:pipeline:${name}:job-deps:${key}"; - value = mkJobDeps {inherit key job;}; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - customMapAttrs (key: job: { - name = "gitlab-ci:pipeline:${name}:job:${key}"; - value = mkJobRun { - inherit key job; - jobDeps = jobsMappedForDeps."gitlab-ci:pipeline:${name}:job-deps:${key}"; - }; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - customMapAttrs (key: job: { - name = key; - value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - mkJobPatched { - inherit key job; - pipeline_name = name; - }; - }) - jobs; - in { - packages = - # gitlab-ci:pipeline: - # gitlab-ci:pipeline::job: - # gitlab-ci:pipeline::job-deps: - { - "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-${name}.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript; - finalConfig = rest // jobsPatched; - }; -} diff --git a/lib/sandbox_helper.sh b/lib/sandbox_helper.sh deleted file mode 100644 index 4ea5ec5..0000000 --- a/lib/sandbox_helper.sh +++ /dev/null @@ -1,73 +0,0 @@ -echo -e "\e[32mSetting up...\e[0m" - -actualJobScript=$1 -shift - -INCLUDE_DIRTY=false -NO_SANDBOX=false -KEEP_TMP=false -KEEP_ENV="" -# parse flags -while [[ $# -gt 0 ]]; do - case "$1" in - --include-dirty) - INCLUDE_DIRTY=true - shift - ;; - --no-sandbox) - NO_SANDBOX=true - shift - ;; - --keep-tmp) - KEEP_TMP=true - shift - ;; - --keep-env) - KEEP_ENV="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -if [ "$NO_SANDBOX" = false ]; then - echo "Running with simple sandboxing" - if [ "$KEEP_TMP" = false ]; then - trap "rm -rf '$TMPDIR'" EXIT - else - echo "Temp dir will be preserved at: $TMPDIR" - fi - - # check if dirty - DIRTY_PATCH="" - if ! git diff --quiet && ! git diff --staged --quiet; then - echo "Warning: working tree is dirty." - DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch") - git diff --staged > "$DIRTY_PATCH" - trap "rm -f '$DIRTY_PATCH'" EXIT - fi - TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX") - git clone . $TMPDIR - pushd $TMPDIR >/dev/null - if [[ ! -z "$DIRTY_PATCH" && "$INCLUDE_DIRTY" = true ]]; then - echo "Copying dirty changes..." - git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes" - fi - - echo "Running job in $TMPDIR" - env -i $( - if [[ -n "$KEEP_ENV" ]]; then - IFS=',' read -ra VARS <<< "$KEEP_ENV" - for var in "${VARS[@]}"; do - printf '%s=%q ' "$var" "${!var}" - done - fi - ) bash $actualJobScript - popd >/dev/null -else - exec $actualJobScript -fi - diff --git a/lib/utils.nix b/lib/utils.nix index ae15284..a40dadd 100644 --- a/lib/utils.nix +++ b/lib/utils.nix @@ -1,46 +1,46 @@ -{pkgs, ...}: { - commitAndPushFiles = { - message, - files ? [], - }: jobArgs: - jobArgs - // { - before_script = - (jobArgs.before_script or []) - ++ [ - # sh - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" - eval "$(ssh-agent -s)" >/dev/null; - mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; - ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; - echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; - git config --global user.email "$GIT_EMAIL" >/dev/null; - git config --global user.name "$GIT_NAME" >/dev/null; - export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; - git remote rm origin && git remote add origin ''${CI_PUSH_REPO} - echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" - '' - ]; - script = let - addScript = - if builtins.length files == 0 - then "" - else "git add ${builtins.concatStringsSep " " files}"; - in - (jobArgs.script or []) - ++ [ - # sh - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" - ${addScript} - git diff --cached --exit-code >/dev/null && - echo "Nothing to commit" || - git commit -m "${message}" --no-verify; - git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip - echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" - '' - ]; - nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; - }; +{ + mkUtils = {pkgs, ...}: { + commitAndPushFiles = { + message, + files ? [], + }: jobArgs: + jobArgs + // { + before_script = + (jobArgs.before_script or []) + ++ [ + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" + eval "$(ssh-agent -s)" >/dev/null; + mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; + ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; + echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; + git config --global user.email "$GIT_EMAIL" >/dev/null; + git config --global user.name "$GIT_NAME" >/dev/null; + export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; + git remote rm origin && git remote add origin ''${CI_PUSH_REPO} + echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" + '' + ]; + script = let + addScript = + if builtins.length files == 0 + then "" + else "git add ${builtins.concatStringsSep " " files}"; + in + (jobArgs.script or []) + ++ [ + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" + ${addScript} + git diff --cached --exit-code >/dev/null && + echo "Nothing to commit" || + git commit -m "${message}" --no-verify; + git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip + echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" + '' + ]; + nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; + }; + }; } diff --git a/snapshots/default.snap.json b/snapshots/default.snap.json deleted file mode 100644 index c247a78..0000000 --- a/snapshots/default.snap.json +++ /dev/null @@ -1 +0,0 @@ -{"docs":{"after_script":["finalize_nix_ci"],"artifacts":{"paths":["public"]},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:docs\""],"image":"$NIX_CI_IMAGE","script":["nix build .#docs:default\nmkdir -p public\ncp -r result/. public/\n"],"stage":"build"},"nixtest":{"after_script":["finalize_nix_ci"],"allow_failure":true,"artifacts":{"reports":{"junit":"junit.xml"},"when":"always"},"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:nixtest\""],"image":"$NIX_CI_IMAGE","script":["nix run .#nixtests:run -- --junit=junit.xml --pure"],"stage":"nixtest"},"pages":{"artifacts":{"paths":["public"]},"image":"alpine:latest","rules":[{"if":"$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"}],"script":["true"],"stage":"deploy"},"stages":["test","nixtest","build","deploy"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test\""],"cache":[{"key":"$CI_JOB_NAME-$CI_COMMIT_REF_SLUG","paths":[".nix-cache/"]}],"image":"$NIX_CI_IMAGE","script":["hello","curl google.de","echo $TEST $TEST_WITH_DERIVATION"],"stage":"test","variables":{"NIX_CI_CACHE_STRATEGY":"runner","TEST":"test"}},"test-default":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:default:job-deps:test-default\""],"image":"$NIX_CI_IMAGE","script":["hello"],"stage":"test"},"test-non-nix":{"image":"alpine:latest","script":["echo \"This job will not be modified to use nix\""],"stage":"test"}} \ No newline at end of file diff --git a/snapshots/non-default.snap.json b/snapshots/non-default.snap.json deleted file mode 100644 index 9ed3256..0000000 --- a/snapshots/non-default.snap.json +++ /dev/null @@ -1 +0,0 @@ -{"stages":["test"],"test":{"after_script":["finalize_nix_ci"],"before_script":["source setup_nix_ci \"gitlab-ci:pipeline:non-default:job-deps:test\""],"image":"$NIX_CI_IMAGE","script":["echo Hello from another pipeline"],"stage":"test"}} \ No newline at end of file diff --git a/tests/ci-lib.nix b/tests/ci-lib.nix deleted file mode 100644 index 577139c..0000000 --- a/tests/ci-lib.nix +++ /dev/null @@ -1,140 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./../lib {inherit lib pkgs;}; -in { - nixtest.suites."CI Lib" = { - pos = __curPos; - tests = let - inherit (cilib.jobDeps) mkJobDeps; - inherit (cilib.jobRun) mkJobRun; - inherit (cilib.jobPatch) mkJobPatched; - inherit (cilib.pipeline) mkPipeline; - deps = mkJobDeps { - key = "test"; - job = { - nix.deps = [pkgs.hello]; - variables.TEST = "${pkgs.curl}"; - }; - }; - in [ - { - name = "jobDeps"; - type = "script"; - script = - # sh - '' - export PATH=${lib.makeBinPath [pkgs.gnugrep]} - grep -q "/nix/store" ${deps} - grep -q 'hello/bin:$PATH' ${deps} - grep -q "export TEST=" ${deps} - grep -q "curl" ${deps} - ''; - } - { - name = "jobRun"; - type = "script"; - script = let - run = mkJobRun { - key = "test"; - job.script = ["hello"]; - jobDeps = deps; - }; - in - # sh - '' - export PATH=${lib.makeBinPath [pkgs.gnugrep]} - grep -q "sandbox-helper" ${run}/bin/gitlab-ci-job:test - grep -q "gitlab-ci-job-test-raw" ${run}/bin/gitlab-ci-job:test - grep -q "gitlab-ci-job-deps-test" ${run.passthru.actualJobScript} - grep -q "Running script..." ${run.passthru.actualJobScript} - grep -q "hello" ${run.passthru.actualJobScript} - ''; - } - { - name = "jobPatched nix disabled"; - expected = {}; - actual = mkJobPatched { - key = "test"; - pipeline_name = "test"; - job.nix.enable = false; - }; - } - { - name = "jobPatched without runner cache"; - expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - }; - actual = mkJobPatched { - key = "test"; - pipeline_name = "test"; - job.nix = { - enable = true; - enable-runner-cache = false; - }; - }; - } - { - name = "jobPatched with runner cache"; - expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - cache = [ - { - key = "test"; - paths = [".nix-cache/"]; - } - ]; - variables."NIX_CI_CACHE_STRATEGY" = "runner"; - }; - actual = mkJobPatched { - key = "test"; - pipeline_name = "test"; - job.nix = { - enable = true; - enable-runner-cache = true; - runner-cache-key = "test"; - }; - }; - } - { - name = "mkPipeline empty"; - expected = {}; - actual = - (mkPipeline { - name = "test"; - pipeline.jobs = {}; - }).finalConfig; - } - { - name = "mkPipeline empty packages"; - type = "script"; - script = let - pipeline = builtins.toFile "pipeline-test" (builtins.toJSON - (mkPipeline { - name = "test"; - pipeline.jobs = {}; - }).packages); - in - # sh - '' - set -euo pipefail - export PATH=${lib.makeBinPath [pkgs.jq pkgs.gnugrep pkgs.coreutils]} - # single key - jq 'keys | length == 1' "${pipeline}" | grep -q true - # key is exactly "gitlab-ci:pipeline:test" - jq -r 'keys[0]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test" - # value contains "/nix/store/" - jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "/nix/store/" - # value contains "gitlab-ci-test.yml" - jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "gitlab-ci-test.yml" - # file only contains "{}" - [[ "$(cat $(jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}"))" == "{}" ]] - ''; - } - ]; - }; -} diff --git a/tests/default.nix b/tests/default.nix deleted file mode 100644 index 764ae46..0000000 --- a/tests/default.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ - imports = [ - ./utils.nix - ./ci-lib.nix - ./helpers.nix - ./pipeline-yamls.nix - ]; -} diff --git a/tests/helpers.nix b/tests/helpers.nix deleted file mode 100644 index 1d100fc..0000000 --- a/tests/helpers.nix +++ /dev/null @@ -1,96 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./../lib {inherit lib pkgs;}; -in { - nixtest.suites."Helpers" = { - pos = __curPos; - tests = let - inherit (cilib) helpers; - in [ - { - name = "appendToAfterScript nix disabled"; - expected = {}; - actual = helpers.appendToAfterScript [] {}; - } - { - name = "appendToAfterScript empty"; - expected = { - nix.enable = true; - after_script = []; - }; - actual = helpers.appendToAfterScript [] {nix.enable = true;}; - } - { - name = "appendToAfterScript"; - expected = { - nix.enable = true; - after_script = ["echo after_script" "finalize_nix_ci"]; - }; - actual = helpers.appendToAfterScript ["finalize_nix_ci"] { - nix.enable = true; - after_script = ["echo after_script"]; - }; - } - { - name = "prependToBeforeScript nix disabled"; - expected = {}; - actual = helpers.prependToBeforeScript [] {}; - } - { - name = "prependToBeforeScript empty"; - expected = { - nix.enable = true; - before_script = []; - }; - actual = helpers.prependToBeforeScript [] {nix.enable = true;}; - } - { - name = "prependToBeforeScript"; - expected = { - nix.enable = true; - before_script = ["setup_nix_ci" "echo before_script"]; - }; - actual = helpers.prependToBeforeScript ["setup_nix_ci"] { - nix.enable = true; - before_script = ["echo before_script"]; - }; - } - { - name = "toYaml"; - expected = ''{"hello":"world"}''; - actual = builtins.readFile (helpers.toYaml "test" {hello = "world";}); - } - { - name = "filterAttrsRec"; - expected = {world = "world";}; - actual = helpers.filterAttrsRec (n: v: v != null) { - hello = null; - world = "world"; - }; - } - { - name = "filterJobVariables with store paths"; - expected = {HELLO = "${pkgs.hello}";}; - actual = helpers.filterJobVariables true { - variables = { - HELLO = "${pkgs.hello}"; - WORLD = "world"; - }; - }; - } - { - name = "filterJobVariables without store paths"; - expected = {WORLD = "world";}; - actual = helpers.filterJobVariables false { - variables = { - HELLO = "${pkgs.hello}"; - WORLD = "world"; - }; - }; - } - ]; - }; -} diff --git a/tests/pipeline-yamls.nix b/tests/pipeline-yamls.nix deleted file mode 100644 index 32effbe..0000000 --- a/tests/pipeline-yamls.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ - lib, - pkgs, - self', - ... -}: let - cilib = import ./../lib {inherit lib pkgs;}; -in { - nixtest.suites."Pipeline YAMLs" = { - pos = __curPos; - tests = let - jsonFile = file: builtins.fromJSON (builtins.readFile file); - in [ - { - name = "default"; - type = "snapshot"; - actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:default"; - } - { - name = "non-default"; - type = "snapshot"; - actual = jsonFile self'.legacyPackages."gitlab-ci:pipeline:non-default"; - } - ]; - }; -} diff --git a/tests/utils.nix b/tests/utils.nix deleted file mode 100644 index 8d9591f..0000000 --- a/tests/utils.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ - lib, - pkgs, - ... -}: let - cilib = import ./../lib {inherit lib pkgs;}; -in { - nixtest.suites."Utils" = { - pos = __curPos; - tests = [ - { - name = "commitAndPushFiles"; - type = "script"; - script = let - inherit (cilib) utils; - job = builtins.toFile "test" ( - builtins.unsafeDiscardStringContext ( - builtins.toJSON ( - utils.commitAndPushFiles { - message = "hello world"; - files = ["a.md" "b.txt"]; - } {} - ) - ) - ); - in - # sh - '' - export PATH=${lib.makeBinPath [pkgs.gnugrep]} - grep -q 'git commit -m \\"hello world\\"' ${job} - grep -q 'git add a.md b.txt' ${job} - ''; - } - ]; - }; -}