mirror of
https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git
synced 2025-12-12 02:00:13 +01:00
feat(v2): initial v2 implementation
add multi-arch (arm & x64) image add multiple pipelines (ci now creates the "default" pipeline as a shorthand) simplify devenv flake input merge all cache options together, now $NIX_CI_CACHE_STRATEGY decides how the cache works setup_nix_ci and finalize_nix_ci are now flake packages and work standalone the specific image is not needed anymore, any image with the right dependencies works runner cache is not the default anymore (because it sucked most of the time) the pipeline is selected by $NIX_CI_PIPELINE_NAME or if empty by $CI_PIPELINE_SOURCE, so for the old behaviour $NIX_CI_PIPELINE_NAME=default is needed, future work will be needed to handle this more nicely
This commit is contained in:
parent
016e6c9dc7
commit
586fb88b9d
6 changed files with 409 additions and 412 deletions
240
flake.nix
240
flake.nix
|
|
@ -39,16 +39,15 @@
|
|||
};
|
||||
};
|
||||
};
|
||||
# should set the "default" pipeline
|
||||
ci = {
|
||||
# use the image built in the parent pipeline for dogfooding
|
||||
config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA";
|
||||
stages = ["test"];
|
||||
jobs = {
|
||||
"test" = {
|
||||
stage = "test";
|
||||
nix = {
|
||||
deps = [pkgs.hello pkgs.curl];
|
||||
disable-cache = false;
|
||||
enable-runner-cache = true;
|
||||
};
|
||||
variables = {
|
||||
TEST = "test";
|
||||
|
|
@ -60,6 +59,11 @@
|
|||
"echo $TEST $TEST_WITH_DERIVATION"
|
||||
];
|
||||
};
|
||||
"test-default" = {
|
||||
stage = "test";
|
||||
nix.deps = [pkgs.hello];
|
||||
script = ["hello"];
|
||||
};
|
||||
"test-non-nix" = {
|
||||
nix.enable = false;
|
||||
stage = "test";
|
||||
|
|
@ -70,130 +74,130 @@
|
|||
};
|
||||
};
|
||||
};
|
||||
pipelines."non-default" = {
|
||||
stages = ["test"];
|
||||
jobs = {
|
||||
"test" = {
|
||||
stage = "test";
|
||||
script = [
|
||||
"echo Hello from another pipeline"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
packages = let
|
||||
setupScript = extra_setup:
|
||||
pkgs.writeShellScriptBin "setup_nix_ci" ''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI"
|
||||
nix path-info --all > /tmp/nix-store-before
|
||||
setupScript = pkgs.writeShellScriptBin "setup_nix_ci" ''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:nix_setup\\r\\e[0KSetting up Nix CI"
|
||||
nix path-info --all > /tmp/nix-store-before
|
||||
|
||||
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
|
||||
${extra_setup}
|
||||
else
|
||||
echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration"
|
||||
fi
|
||||
|
||||
export NIX_CONFIG="
|
||||
extra-trusted-public-keys = $NIX_PUBLIC_KEYS
|
||||
extra-trusted-substituters = $NIX_SUBSTITUTERS
|
||||
extra-substituters = $NIX_SUBSTITUTERS
|
||||
$NIX_CONFIG
|
||||
$NIX_EXTRA_CONFIG
|
||||
"
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"
|
||||
if [ -z "$_NIX_CI_DISABLE_CACHE" ]; then
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($_NIX_CI_CACHE_STRATEGY)"
|
||||
case "$_NIX_CI_CACHE_STRATEGY" in
|
||||
"runner")
|
||||
export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"}
|
||||
echo "Runner Cache: $RUNNER_CACHE"
|
||||
export NIX_CONFIG="$NIX_CONFIG
|
||||
extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true
|
||||
extra-substituters = $RUNNER_CACHE?priority=10&trusted=true
|
||||
"
|
||||
;;
|
||||
"attic")
|
||||
echo "Attic Cache: $ATTIC_CACHE"
|
||||
attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true
|
||||
attic use "$ATTIC_CACHE" || true
|
||||
;;
|
||||
"cachix")
|
||||
echo "Cachix Cache: $CACHIX_CACHE"
|
||||
cachix use "$CACHIX_CACHE" || true
|
||||
;;
|
||||
"none")
|
||||
echo "Cache strategy is none, doing nothing..."
|
||||
;;
|
||||
*)
|
||||
echo "WARNING: Invalid cache strategy set: '$_NIX_CI_CACHE_STRATEGY'"
|
||||
;;
|
||||
esac
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K"
|
||||
else
|
||||
echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..."
|
||||
fi
|
||||
|
||||
# load the job's deps only if the name was passed
|
||||
if [[ ! -z $1 ]]; then
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job"
|
||||
nix build .#gitlab-ci-job-deps:$1
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job"
|
||||
nix build .#$1
|
||||
source $(readlink -f result)
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K"
|
||||
fi
|
||||
'';
|
||||
finalizeScript = push_command:
|
||||
pkgs.writeShellScriptBin "finalize_nix_ci" ''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache"
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"
|
||||
'';
|
||||
finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" ''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci\\r\\e[0KFinalizing Nix CI..."
|
||||
nix path-info --all > /tmp/nix-store-after
|
||||
${pkgs.diffutils}/bin/diff --new-line-format="%L" \
|
||||
echo "Finding new paths..."
|
||||
NEW_PATHS=$(${pkgs.diffutils}/bin/diff --new-line-format="%L" \
|
||||
--old-line-format="" --unchanged-line-format="" \
|
||||
/tmp/nix-store-before /tmp/nix-store-after \
|
||||
| {
|
||||
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
|
||||
${push_command}
|
||||
else
|
||||
${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; }
|
||||
fi
|
||||
}
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
|
||||
'';
|
||||
mkImage = extraPackages:
|
||||
pkgs.dockerTools.buildImage {
|
||||
name = "nix-gitlab-ci";
|
||||
fromImage = pkgs.dockerTools.pullImage {
|
||||
imageName = "nixpkgs/nix-flakes";
|
||||
imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c";
|
||||
sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf";
|
||||
finalImageName = "nixpkgs/nix-flakes";
|
||||
finalImageTag = "latest";
|
||||
};
|
||||
copyToRoot = pkgs.buildEnv {
|
||||
name = "image-root";
|
||||
paths =
|
||||
[
|
||||
pkgs.gitMinimal
|
||||
pkgs.gnugrep
|
||||
]
|
||||
++ extraPackages;
|
||||
pathsToLink = ["/bin"];
|
||||
};
|
||||
};
|
||||
/tmp/nix-store-before /tmp/nix-store-after)
|
||||
COUNT=$(${pkgs.busybox}/bin/wc -l <<<"$NEW_PATHS")
|
||||
if [ -z "$_NIX_CI_DISABLE_CACHE" ]; then
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($_NIX_CI_CACHE_STRATEGY)"
|
||||
echo $NEW_PATHS | {
|
||||
case "$_NIX_CI_CACHE_STRATEGY" in
|
||||
"runner")
|
||||
export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"}
|
||||
# add ^* to all store paths ending in .drv (prevent warning log spam)
|
||||
${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true
|
||||
;;
|
||||
"attic")
|
||||
attic push --stdin ci:$ATTIC_CACHE || true
|
||||
;;
|
||||
"cachix")
|
||||
cachix push $CACHIX_CACHE || true
|
||||
;;
|
||||
"none")
|
||||
echo "Cache strategy is none, doing nothing..."
|
||||
;;
|
||||
*)
|
||||
echo "WARNING: Invalid cache strategy set: '$_NIX_CI_CACHE_STRATEGY'"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
|
||||
else
|
||||
echo "Caching disabled, not uploading $COUNT new store entries..."
|
||||
fi
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K"
|
||||
'';
|
||||
in {
|
||||
setup-script =
|
||||
setupScript
|
||||
# sh
|
||||
''
|
||||
# extra_setup
|
||||
true
|
||||
'';
|
||||
finalize-script =
|
||||
finalizeScript
|
||||
# sh
|
||||
''
|
||||
# push_command
|
||||
true
|
||||
'';
|
||||
image = mkImage [
|
||||
(setupScript
|
||||
# sh
|
||||
''
|
||||
cachedir="$(pwd)/.nix-cache"
|
||||
echo "Configuring caching with the Runner Cache in $cachedir..."
|
||||
export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true"
|
||||
'')
|
||||
(finalizeScript
|
||||
# sh
|
||||
''
|
||||
# add ^* to all store paths ending in .drv (prevent warning log spam)
|
||||
${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true
|
||||
'')
|
||||
];
|
||||
image-cachix = mkImage [
|
||||
(setupScript
|
||||
# sh
|
||||
''
|
||||
echo "Configuring caching with cachix..."
|
||||
${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true
|
||||
'')
|
||||
(finalizeScript
|
||||
# sh
|
||||
''
|
||||
${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true
|
||||
'')
|
||||
];
|
||||
image-attic = mkImage [
|
||||
(setupScript
|
||||
# sh
|
||||
''
|
||||
echo "Configuring caching with attic..."
|
||||
${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true
|
||||
${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true
|
||||
'')
|
||||
(finalizeScript
|
||||
# sh
|
||||
''
|
||||
${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true
|
||||
'')
|
||||
];
|
||||
setup-script = setupScript;
|
||||
finalize-script = finalizeScript;
|
||||
image = pkgs.dockerTools.buildImage {
|
||||
name = "nix-ci";
|
||||
fromImage = pkgs.dockerTools.pullImage {
|
||||
imageName = "nixpkgs/nix-flakes";
|
||||
# nix run nixpkgs#nix-prefetch-docker -- --image-name nixpkgs/nix-flakes --image-tag latest --arch <amd64/arm64> --os linux
|
||||
imageDigest = "sha256:95bce4317c15dfab3babac5a6d19d3ed41e31a02a8aaf3d4f6639778cb763b0a";
|
||||
sha256 =
|
||||
if pkgs.stdenv.hostPlatform.isAarch64
|
||||
then "DMlSaP+ZVqxd9NxdFydGyfkuJdmOW5jt5iM/7cDyTEM="
|
||||
else "mfTNlGOpThanLlLQ2lL1RTcHqZJWdqUafYDZMeZPWEk=";
|
||||
finalImageName = "nixpkgs/nix-flakes";
|
||||
finalImageTag = "latest";
|
||||
};
|
||||
copyToRoot = pkgs.buildEnv {
|
||||
name = "image-root";
|
||||
paths = [
|
||||
pkgs.gitMinimal
|
||||
pkgs.gnugrep
|
||||
pkgs.cachix
|
||||
pkgs.attic-client
|
||||
setupScript
|
||||
finalizeScript
|
||||
];
|
||||
pathsToLink = ["/bin"];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
checks = packages;
|
||||
|
|
@ -206,11 +210,7 @@
|
|||
# flake & devenv related
|
||||
flake-parts.url = "github:hercules-ci/flake-parts";
|
||||
systems.url = "github:nix-systems/default-linux";
|
||||
devenv = {
|
||||
url = "github:cachix/devenv";
|
||||
inputs.git-hooks.follows = "git-hooks";
|
||||
};
|
||||
git-hooks.url = "github:cachix/git-hooks.nix";
|
||||
devenv.url = "github:cachix/devenv";
|
||||
treefmt-nix.url = "github:numtide/treefmt-nix";
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue