mirror of
https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git
synced 2025-12-12 02:00:13 +01:00
feat: improve caching
This commit is contained in:
parent
a2a7f7c4ed
commit
78e6c5e278
4 changed files with 149 additions and 47 deletions
20
README.md
20
README.md
|
|
@ -1,9 +1,10 @@
|
|||
# Nix Gitlab CI
|
||||
|
||||
Flake module which allows generating a `.gitlab-ci.yml` from Nix.
|
||||
|
||||
This allows easily using any Nix package in CI.
|
||||
Also makes it possible to split CI parts in a separate module
|
||||
which can be imported in multiple projects.
|
||||
|
||||
Also makes it possible to split CI parts in a separate module which can be imported in multiple projects.
|
||||
|
||||
## Usage
|
||||
|
||||
|
|
@ -25,7 +26,7 @@ which can be imported in multiple projects.
|
|||
jobs = {
|
||||
"test" = {
|
||||
stage = "test";
|
||||
deps = [pkgs.unixtools.ping];
|
||||
nix.deps = [pkgs.unixtools.ping];
|
||||
script = [
|
||||
"ping -c 5 8.8.8.8"
|
||||
];
|
||||
|
|
@ -47,3 +48,16 @@ include:
|
|||
image_tag: latest-cachix
|
||||
```
|
||||
|
||||
## Utilities
|
||||
|
||||
### Disable Caching temporarily
|
||||
|
||||
To disable any of the provided caches for a pipeline one can set `NIX_CI_DISABLE_CACHE` to
|
||||
anything non-empty (eg. "yes") when triggering the pipeline.
|
||||
|
||||
The `build:nix-ci` job has a different special environment variable `NIX_CI_SKIP_CACHE`
|
||||
(useful if the generated pipeline is outdated but caching should generally still take place).
|
||||
|
||||
## Thanks to
|
||||
|
||||
Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix
|
||||
|
|
|
|||
42
flake.nix
42
flake.nix
|
|
@ -53,7 +53,10 @@
|
|||
# wait an hour so the image builds
|
||||
when = "delayed";
|
||||
start_in = "1 hour";
|
||||
nix = {
|
||||
deps = [pkgs.hello pkgs.curl];
|
||||
disable-cache = false;
|
||||
};
|
||||
variables = {
|
||||
TEST = "test";
|
||||
TEST_WITH_DERIVATION = "${pkgs.hello}/test";
|
||||
|
|
@ -65,7 +68,7 @@
|
|||
];
|
||||
};
|
||||
"test-non-nix" = {
|
||||
nix = false;
|
||||
nix.enable = false;
|
||||
stage = "test";
|
||||
image = "alpine:latest";
|
||||
script = [
|
||||
|
|
@ -80,12 +83,18 @@
|
|||
pkgs.writeShellScriptBin "setup_nix_ci" ''
|
||||
echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI"
|
||||
nix path-info --all > /tmp/nix-store-before
|
||||
|
||||
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
|
||||
${extra_setup}
|
||||
export NIX_CONF="
|
||||
extra-trusted-public-keys = $NIX_PUBLIC_KEYS \n
|
||||
extra-trusted-substituters = $NIX_SUBSTITUTERS \n
|
||||
extra-substituters = $NIX_SUBSTITUTERS \n
|
||||
$NIX_EXTRA_CONF
|
||||
else
|
||||
echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration"
|
||||
fi
|
||||
|
||||
export NIX_CONFIG="
|
||||
extra-trusted-public-keys = $NIX_PUBLIC_KEYS
|
||||
extra-trusted-substituters = $NIX_SUBSTITUTERS
|
||||
extra-substituters = $NIX_SUBSTITUTERS
|
||||
$NIX_EXTRA_CONFIG
|
||||
"
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"
|
||||
${
|
||||
|
|
@ -104,7 +113,14 @@
|
|||
nix path-info --all > /tmp/nix-store-after
|
||||
${pkgs.diffutils}/bin/diff --new-line-format="%L" \
|
||||
--old-line-format="" --unchanged-line-format="" \
|
||||
/tmp/nix-store-before /tmp/nix-store-after | ${push_command}
|
||||
/tmp/nix-store-before /tmp/nix-store-after \
|
||||
| {
|
||||
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
|
||||
${push_command}
|
||||
else
|
||||
${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; }
|
||||
fi
|
||||
}
|
||||
echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
|
||||
'';
|
||||
mkImage = extraPackages:
|
||||
|
|
@ -129,13 +145,19 @@
|
|||
};
|
||||
};
|
||||
in {
|
||||
setup-script = setupScript "# extra_setup";
|
||||
setup-script = setupScript "true # extra_setup";
|
||||
finalize-script = finalizeScript "true # push_command";
|
||||
image = mkImage [
|
||||
(setupScript ''
|
||||
echo "No caching configured, to enable caching use the respective container image tag"
|
||||
cachedir="$(pwd)/.nix-cache"
|
||||
echo "Configuring caching with the Runner Cache in $cachedir..."
|
||||
export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true"
|
||||
'')
|
||||
(finalizeScript ''
|
||||
while read entry; do
|
||||
nix copy --quiet --to "file://$(pwd)/.nix-cache" $entry || true
|
||||
done
|
||||
'')
|
||||
(finalizeScript ''${pkgs.busybox}/bin/wc -l | { read count; echo "No caching configured, not uploading $count new store entries..."; }'')
|
||||
];
|
||||
image-cachix = mkImage [
|
||||
(setupScript ''
|
||||
|
|
|
|||
|
|
@ -33,19 +33,46 @@
|
|||
nix-jobs-per-default = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Handle jobs nix-based by default or via opt-in (in job set nix = true) if false";
|
||||
description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
|
||||
};
|
||||
disable-cache = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE";
|
||||
};
|
||||
cache-key = mkOption {
|
||||
type = types.str;
|
||||
default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
|
||||
description = "Cache key to use for the nix cache";
|
||||
};
|
||||
};
|
||||
jobType = with lib;
|
||||
subType {
|
||||
# nix ci opts
|
||||
nix = mkOption {
|
||||
type = subType {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = cfg.nix-jobs-per-default;
|
||||
description = "Handle this job as a nix job";
|
||||
};
|
||||
deps = mkOption {
|
||||
type = types.listOf types.package;
|
||||
default = [];
|
||||
description = "Dependencies/packages to install for this job";
|
||||
};
|
||||
disable-cache = mkOption {
|
||||
type = types.bool;
|
||||
default = cfg.disable-cache;
|
||||
description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE";
|
||||
};
|
||||
cache-key = mkOption {
|
||||
type = types.str;
|
||||
default = cfg.cache-key;
|
||||
description = "Cache key to use for the nix cache";
|
||||
};
|
||||
};
|
||||
description = "Configure Nix Gitlab CI for each job individually";
|
||||
};
|
||||
# gitlab opts
|
||||
script = mkOption {
|
||||
|
|
@ -64,7 +91,7 @@
|
|||
allow_failure = mkNullOption (types.either types.attrs types.bool);
|
||||
artifacts = mkNullOption (types.attrs);
|
||||
before_script = mkNullOption (types.listOf types.str);
|
||||
cache = mkNullOption (types.attrs);
|
||||
cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
|
||||
coverage = mkNullOption (types.str);
|
||||
dependencies = mkNullOption (types.listOf types.str);
|
||||
environment = mkNullOption (types.either types.attrs types.str);
|
||||
|
|
@ -124,7 +151,7 @@
|
|||
mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
|
||||
prepend = key: arr: job:
|
||||
job
|
||||
// lib.optionalAttrs job.nix {
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
${key} =
|
||||
arr
|
||||
++ job.${key} or [];
|
||||
|
|
@ -143,11 +170,9 @@
|
|||
variablesWithStorePaths =
|
||||
lib.concatMapAttrs (
|
||||
name: value:
|
||||
if lib.hasInfix "/nix/store/" value
|
||||
then {
|
||||
lib.optionalAttrs (lib.hasInfix "/nix/store/" value) {
|
||||
${name} = value;
|
||||
}
|
||||
else {}
|
||||
)
|
||||
(job.variables or {});
|
||||
variableExports = lib.concatMapStrings (x: "${x}\n") (
|
||||
|
|
@ -156,7 +181,7 @@
|
|||
in {
|
||||
name = "gitlab-ci-job-deps:${key}";
|
||||
value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" ''
|
||||
export PATH="${lib.makeBinPath job.deps}:$PATH";
|
||||
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
|
||||
${variableExports}
|
||||
'';
|
||||
})
|
||||
|
|
@ -181,17 +206,32 @@
|
|||
"finalize_nix_ci"
|
||||
]
|
||||
job))
|
||||
// lib.optionalAttrs job.nix {
|
||||
// lib.optionalAttrs job.nix.enable {
|
||||
image = job.image;
|
||||
variables = lib.concatMapAttrs (name: value:
|
||||
if lib.hasInfix "/nix/store/" value
|
||||
then {}
|
||||
else {
|
||||
variables =
|
||||
lib.concatMapAttrs (name: value:
|
||||
lib.optionalAttrs (!lib.hasInfix "/nix/store/" value) {
|
||||
${name} = value;
|
||||
})
|
||||
(job.variables or {});
|
||||
(job.variables or {})
|
||||
// lib.optionalAttrs job.nix.disable-cache {
|
||||
NIX_CI_DISABLE_CACHE = "yes";
|
||||
};
|
||||
cache =
|
||||
(
|
||||
let
|
||||
c = job.cache or [];
|
||||
in
|
||||
if builtins.isList c
|
||||
then c
|
||||
else [c]
|
||||
)
|
||||
++ (lib.optional (!job.nix.disable-cache) {
|
||||
key = job.nix.cache-key;
|
||||
paths = [".nix-cache/"];
|
||||
});
|
||||
}
|
||||
) ["nix" "deps"];
|
||||
) ["nix"];
|
||||
})
|
||||
jobs;
|
||||
in
|
||||
|
|
|
|||
|
|
@ -4,28 +4,53 @@ spec:
|
|||
type: string
|
||||
description: "latest | latest-cachix | latest-attic etc."
|
||||
default: latest
|
||||
cache_files:
|
||||
type: array
|
||||
description: |
|
||||
Files to use as the cache key for the generated pipeline yaml.
|
||||
If you use "ci.nix" to define CI, add that here for example
|
||||
default: ["flake.nix", "flake.lock"]
|
||||
disable_cache:
|
||||
type: string
|
||||
description: |
|
||||
Disables any caching provided by this component. Set to any non-empty value to disable caching.
|
||||
default: ""
|
||||
---
|
||||
|
||||
stages:
|
||||
- build
|
||||
- trigger
|
||||
|
||||
variables:
|
||||
NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]"
|
||||
nix-ci:build:
|
||||
stage: build
|
||||
image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]]
|
||||
cache:
|
||||
- key:
|
||||
files: $[[ inputs.cache_files ]]
|
||||
paths:
|
||||
- generated-gitlab-ci.yml
|
||||
- key: nix
|
||||
paths:
|
||||
- .nix-cache/
|
||||
before_script:
|
||||
- source setup_nix_ci
|
||||
# generated-gitlab-ci.yml exists in the cache
|
||||
- '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "Using cached pipeline file (skip cache with NIX_CI_SKIP_CACHE)" || true'
|
||||
# allow the user to manually skip the cache (when the key files are not correctly configured etc.)
|
||||
- '[ -n "$NIX_CI_SKIP_CACHE" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_SKIP_CACHE)" || true'
|
||||
# only setup when we need to generate the pipeline yaml
|
||||
- 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi'
|
||||
script:
|
||||
# build the generated-gitlab-ci.yml
|
||||
- nix build .#gitlab-ci-config
|
||||
- install result generated-gitlab-ci.yml
|
||||
# build the generated-gitlab-ci.yml if it does not exist in the cache
|
||||
- 'if [ -z "$CACHED" ]; then nix build .#gitlab-ci-config && install result generated-gitlab-ci.yml; fi'
|
||||
after_script:
|
||||
# upload to binary cache
|
||||
- finalize_nix_ci
|
||||
# NOTE: environment variables of before_script and script don't exist here anymore
|
||||
#
|
||||
# save to binary cache or Gitlab CI cache only if we actually built something
|
||||
# check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called
|
||||
- 'if [ -f "/tmp/nix-store-before" ]; then finalize_nix_ci; fi'
|
||||
artifacts:
|
||||
paths:
|
||||
- generated-gitlab-ci.yml
|
||||
|
||||
nix-ci:trigger:
|
||||
stage: trigger
|
||||
needs:
|
||||
|
|
@ -35,4 +60,5 @@ nix-ci:trigger:
|
|||
- artifact: generated-gitlab-ci.yml
|
||||
job: nix-ci:build
|
||||
strategy: depend
|
||||
|
||||
forward:
|
||||
pipeline_variables: true
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue