technofab/nix-gitlab-ci
1
0
Fork 0
mirror of https://gitlab.com/TECHNOFAB/nix-gitlab-ci.git synced 2025-12-12 02:00:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'feat/v2' into 'main'

Browse source 
feat: v2

Closes #6, #7, #10, #13, #17, #19, #20, and #21

See merge request TECHNOFAB/nix-gitlab-ci!10
This commit is contained in:
TECHNOFAB 2025-05-02 17:31:03 +02:00
commit a738869c3b
22 changed files with 1183 additions and 511 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.envrc
View file

@ -1,4 +1 @@
if ! use flake . --impure use flake . --impure --accept-flake-config
then
echo "devenv could not be build. The devenv environment was not loaded. Make the necessary changes to flake.nix and hit enter to try again." >&2
fi

40
.gitlab-ci.yml
View file

@ -1,7 +1,7 @@
include: include:
- component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHA - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHA
inputs: inputs:
image_tag: $CI_COMMIT_SHORT_SHA version: $CI_COMMIT_SHORT_SHA
stages: stages:
- build-images - build-images
- build - build
@ -10,26 +10,40 @@ build:image:
stage: build-images stage: build-images
parallel: parallel:
matrix: matrix:
- VARIANT: ["", "-cachix", "-attic"] - ARCH: ["x86_64-linux", "aarch64-linux"]
image: nixpkgs/nix-flakes:latest image: nixpkgs/nix-flakes:latest
before_script:
- nix profile install nixpkgs#skopeo
- export PATH="$PATH:$HOME/.nix-profile/bin"
script: script:
- nix build .#image${VARIANT} - nix build .#image --system $ARCH
after_script:
- install -D result dist/nix-ci-$ARCH.tar.gz
artifacts:
paths:
- dist
deploy:image:
stage: build-images
image: nixpkgs/nix-flakes:latest
needs:
- build:image
before_script:
- nix profile install nixpkgs#buildah
- export PATH="$PATH:$HOME/.nix-profile/bin"
- export REGISTRY_AUTH_FILE=''${HOME}/auth.json
- echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
- mkdir -p /etc/containers && echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json
- mkdir -p /var/tmp
script:
- export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-} - export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-}
- skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_SHORT_SHA}${VARIANT}" - buildah manifest create localhost/nix-ci
- buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz
- buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz
- buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_SHORT_SHA}
# branches # branches
- | - |
if [ -z "$CI_COMMIT_TAG" ]; then if [ -z "$CI_COMMIT_TAG" ]; then
skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${NORMALIZED_BRANCH/main/latest}
"docker-archive:result" \
"docker://$CI_REGISTRY_IMAGE/nix-ci:${NORMALIZED_BRANCH/main/latest}${VARIANT}";
fi fi
# tags # tags
- | - |
if [ -n "$CI_COMMIT_TAG" ]; then if [ -n "$CI_COMMIT_TAG" ]; then
skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_TAG}
"docker-archive:result" \
"docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_TAG}${VARIANT}";
fi fi

29
README.md
View file

@ -1,4 +1,11 @@
# Nix Gitlab CI # Nix GitLab CI
[![built with nix](https://img.shields.io/static/v1?logo=nixos&logoColor=white&label=&message=Built%20with%20Nix&color=41439a)](https://builtwithnix.org)
[![pipeline status](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/badges/main/pipeline.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/commits/main)
![License: MIT](https://img.shields.io/gitlab/license/technofab/nix-gitlab-ci)
[![Latest Release](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/badges/release.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/releases)
[![Support me](https://img.shields.io/badge/Support-me-orange)](https://tec.tf/#support)
[![Docs](https://img.shields.io/badge/Read-Docs-orange)](https://nix-gitlab-ci.projects.tf)
Flake module which allows generating a `.gitlab-ci.yml` from Nix. Flake module which allows generating a `.gitlab-ci.yml` from Nix.
@ -12,7 +19,7 @@ Also makes it possible to split CI parts in a separate module which can be impor
# flake.nix # flake.nix
{ {
... ...
inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/<version>?dir=lib"; # recommendation: pin to the latest release/version
outputs = {...}: flake-parts.lib.mkFlake {...} { outputs = {...}: flake-parts.lib.mkFlake {...} {
imports = [ imports = [
@ -21,6 +28,7 @@ Also makes it possible to split CI parts in a separate module which can be impor
... ...
perSystem = {pkgs, ...}: { perSystem = {pkgs, ...}: {
# ci is a shortcut and creates a "default" pipeline
ci = { ci = {
stages = ["test"]; stages = ["test"];
jobs = { jobs = {
@ -33,6 +41,11 @@ Also makes it possible to split CI parts in a separate module which can be impor
}; };
}; };
}; };
# runs on a merge request for example
pipelines."merge_request_event" = {
stages = ["some_stage"];
jobs = { ... };
};
... ...
} }
} }
@ -42,10 +55,9 @@ Also makes it possible to split CI parts in a separate module which can be impor
```yaml ```yaml
# .gitlab-ci.yml # .gitlab-ci.yml
include: include:
- component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@<version> # recommendation: use the latest version (try not to use latest) - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@<version> # recommendation: pin to the latest release/version (don't use "main" etc.)
inputs: inputs:
# specify inputs here, for example: version: <version> # docker image tag, use the same version as a above
image_tag: latest-cachix
``` ```
## Utilities ## Utilities
@ -63,16 +75,15 @@ The `build:nix-ci` job has a different special environment variable `NIX_CI_FORC
You can run any job's script (+ before and after) locally with Nix for easier testing: You can run any job's script (+ before and after) locally with Nix for easier testing:
```sh ```sh
nix run .#gitlab-ci-job:<name> # / pipeline name, like "default"
nix run .#gitlab-ci:pipeline:<pipeline name>:job:<name>
``` ```
There is also `.#gitlab-ci-job-deps:<name>` which generates and exports the required environment variables for each job: There is also `.#gitlab-ci:pipeline:<pipeline name>:job-deps:<name>` which generates and exports the required environment variables for each job:
- PATH (with all deps) - PATH (with all deps)
- any custom env variables which contain store paths to not break stuff when switching archs - any custom env variables which contain store paths to not break stuff when switching archs
Please see #8 for some issues and further improvements on this.
## Thanks to ## Thanks to
Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix

50
docs/caching.md Normal file
View file

@ -0,0 +1,50 @@
# Caching
Nix GitLab CI supports several caching mechanisms to speed up your pipelines.
## GitLab Runner Cache
The runner cache strategy copies the new store paths into a directory `.nix-cache`,
which is then saved in the regular GitLab cache (technically runner cache).
It's also configured as a substituter automatically.
To enable, set the cache strategy to `runner`.
Configure it using these environment variables:
- `RUNNER_CACHE`: path to the runner cache (default `.nix-cache`)
!!! warning
This is very inefficient and should probably only be used for very very small
dependency counts. Otherwise it takes an eternity to save to cache.
## Cachix
Cachix is a hosted binary cache service that can significantly speed up Nix
builds by sharing build results.
To enable, set the cache strategy to `attic`.
Configure it using these environment variables:
- `CACHIX_CACHE`: name of the cache to use
- (`CACHIX_AUTH_TOKEN`): cachix client itself uses this for authentication
!!! warning
Cachix has not been tested. Feedback is appreciated :)
## Attic (Self-Hosted Cache)
Attic is a self-hosted, deduplicating binary cache. It's a great option if you
want more control over your caching infrastructure and to have the cache closer
to your runners.
To enable, set the cache strategy to `attic`.
Configure it using these environment variables:
- `ATTIC_SERVER`: URL of the server
- `ATTIC_CACHE`: name of the cache to use
- `ATTIC_TOKEN`: auth token from the attic server

41
docs/cicd_component.md Normal file
View file

@ -0,0 +1,41 @@
# CI/CD Component
The CI/CD Component has some inputs which configure defaults for Nix GitLab CI.
## `version`
- Type: `string`
Which version of the Nix CI image to use. Using a tag/version is recommended.
Will not do anything if a custom image is specified using `NIX_CI_IMAGE`.
## `cache_strategy`
- Type: `string`
- Default: `"auto"`
- Options: `auto` | `none` | `runner` | `cachix` | `attic`
Sets the default caching strategy.
- `auto`: dynamically selects the best strategy for every job based on env variables
- `none`: disables caching
- `runner`, `cachix` & `attic`: forces every job to use this strategy
Can be overridden by `NIX_CI_CACHE_STRATEGY`, see [Environment Variables](./environment_variables.md).
## `cache_files`
- Type: `array` (of strings)
- Default: `["flake.nix", "flake.lock"]`
Files to use as the cache key for the generated pipeline yaml.
If you use a file like `ci.nix` to define CI, add that here for example.
This makes sure that changes to your Nix CI configuration will invalidate the cache,
otherwise an old pipeline yaml might be used.
!!! warning
The value of this is used in `cache:key:files`, which currently only supports
a max of 2 entries. So use something like `["flake.*", "ci.nix"]` to match
`flake.lock`, `flake.nix` and `ci.nix`.
See [gitlab-org/gitlab#301161](https://gitlab.com/gitlab-org/gitlab/-/issues/301161)

107
docs/environment_variables.md Normal file
View file

@ -0,0 +1,107 @@
# Environment Variables
Nix GitLab CI is mostly controlled using environment variables.
This page outlines all the variables and their use case.
## `NIX_CI_IMAGE`
| | |
| ----------- | -------------------------------------------------------------------------- |
| Default | `registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci@$[[ inputs.version ]]` |
| Description | Image to use for the jobs |
## `NIX_CI_PIPELINE_NAME`
| | |
| ----------- | ------------------------------------------------- |
| Default | N/A |
| Description | Explicitly request a pipeline to be built and ran |
| See also | [Multi Pipeline](./multi_pipeline.md) |
## `NIX_CI_DEFAULT_SOURCES`
| | |
| ----------- | -------------------------------------------------------------------------------------------------------------------------------- |
| Default | `.*` |
| Description | Regex to match `$CI_PIPELINE_SOURCE` against. If it matches, the `default` pipeline will be ran, otherwise `$CI_PIPELINE_SOURCE` |
| See also | [Multi Pipeline](./multi_pipeline.md) |
## `NIX_CI_FORCE_BUILD`
| | |
| ----------- | -------------------------------------------------------------------------------------- |
| Default | N/A |
| Description | Set to any non-empty value to force the `nix-ci:build` job to freshly build the config |
| See also | [Caching](./caching.md) |
## `NIX_CI_DISABLE_CACHE`
| | |
| ----------- | ------------------------------------------------------ |
| Default | N/A |
| Description | Set to any non-empty value to disable caching for jobs |
| See also | [Caching](./caching.md) |
## `NIX_CI_CACHE_STRATEGY`
| | |
| ----------- | --------------------------------------------------------------------------------- |
| Default | `$[[ inputs.cache_strategy ]]` -> defaults to `auto` |
| Description | Caching strategy to use. `auto` will select the strategy based on runner settings |
| See also | [Caching](./caching.md) |
## `NIX_CI_RUNNER_CACHE_STRATEGY`
| | |
| ----------- | ---------------------------------------------------------------- |
| Default | N/A |
| Description | Every runner can set it's own preferred cache strategy with this |
| See also | [Caching](./caching.md) |
## `NIX_CI_DEFAULT_CACHE_STRATEGY`
| | |
| ----------- | ------------------------------------------------------------------------------------------------- |
| Default | `none` |
| Description | If no runner cache strategy is set and the main strategy is set to auto, this will be the default |
| See also | [Caching](./caching.md) |
## `RUNNER_CACHE`
| | |
| ----------- | -------------------------------------- |
| Default | `.nix-cache` |
| Description | Path to directory for the runner cache |
| See also | [Caching](./caching.md) |
## `CACHIX_CACHE`
| | |
| ----------- | ------------------------------- |
| Default | N/A |
| Description | Name of the cachix cache to use |
| See also | [Caching](./caching.md) |
## `ATTIC_CACHE`
| | |
| ----------- | ------------------------------ |
| Default | N/A |
| Description | Name of the attic cache to use |
| See also | [Caching](./caching.md) |
## `ATTIC_SERVER`
| | |
| ----------- | ----------------------- |
| Default | N/A |
| Description | URL of the attic server |
| See also | [Caching](./caching.md) |
## `ATTIC_TOKEN`
| | |
| ----------- | ------------------------------- |
| Default | N/A |
| Description | API token from the attic server |
| See also | [Caching](./caching.md) |

18
docs/examples.md Normal file
View file

@ -0,0 +1,18 @@
# Example Configs
## V2
- [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci)
See `flake.nix` for some random example jobs.
- [TECHNOFAB/nixlets](https://gitlab.com/TECHNOFAB/nixlets)
- [TECHNOFAB/nixmkdocs](https://gitlab.com/TECHNOFAB/nixmkdocs)
- [TECHNOFAB/tofunix](https://gitlab.com/TECHNOFAB/tofunix)
## Old / V1
- [TECHNOFAB/coder-templates](https://gitlab.com/TECHNOFAB/coder-templates)
!!! note
Feel free to edit this page and add your project if you're using
Nix GitLab CI :)

BIN
docs/images/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.9 KiB

BIN
docs/images/logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 40 KiB

11
docs/index.md Normal file
View file

@ -0,0 +1,11 @@
# Nix GitLab CI
This project provides a Nix flake module that allows you to generate your `.gitlab-ci.yml` file directly from your Nix configuration.
## Features
- **Reproducibility:** Leverage Nix's strength in creating reproducible environments for your CI jobs.
- **Easy Dependency Management:** Easily include any package available in Nixpkgs or your own defined packages within your CI jobs using Nix.
- **Modularity:** Define and manage your CI configurations in a structured and modular way using Nix modules, making it easier to share and reuse CI logic across multiple projects.
This documentation will guide you through setting up and using Nix GitLab CI for your projects.

38
docs/kubernetes_runner.md Normal file
View file

@ -0,0 +1,38 @@
# Kubernetes Runner Setup
Using the GitLab Kubernetes runner allows your CI jobs to run as pods in a Kubernetes cluster.
Nix GitLab CI can be integrated with this setup, and using advanced configuration options like
`pod_spec` makes it easy to add runner specific caching.
Using this Runner configuration ...
```toml
[[runners.kubernetes.pod_spec]]
name = "nix-ci-cache-secrets"
patch = '''
containers:
- name: build
envFrom:
- secretRef:
name: nix-ci-cache-env
'''
```
... and a secret containing ...
```yaml
NIX_CI_RUNNER_CACHE_STRATEGY: attic
ATTIC_SERVER: <in-cluster-url> # example: http://atticd.<ns>.svc.cluster.local:8080
ATTIC_CACHE: ci # name however you want, just needs to exist
ATTIC_TOKEN: <token>
```
... makes your jobs automatically cache their Nix store paths to the in-cluster
attic when running with this runner.
Other runners could use cachix or no cache, you get the idea ;P
!!! note
This of course works with any executor where you can set environment
variables. This is just an example how to do it in Kubernetes easily.

31
docs/multi_pipeline.md Normal file
View file

@ -0,0 +1,31 @@
# Multiple Pipelines
With V2, Nix GitLab CI can generate different pipelines, depending on the
pipeline source (`$CI_PIPELINE_SOURCE`).
By default, no matter which source, the `default` pipeline is built and ran.
`$NIX_CI_PIPELINE` can override that, eg. when manually triggering a run.
To configure which source should be 1-to-1 translated to a pipeline with the
same name, set `$NIX_CI_DEFAULT_SOURCES` to a regex which explicitly does not
match these sources. Or set it to an impossible to match regex, then it will
always run the pipeline named after `$CI_PIPELINE_SOURCE`.
## Example 1: always run default
If you only have a single pipeline, you just have to call it `default`.
Everything else works out of the box.
## Example 2: default and merge_request_event
If you want the source `merge_request_event` to trigger a different pipeline,
name it like that and set `$NIX_CI_DEFAULT_SOURCES` to `^(merge_request_event)$`.
Now a merge request will run this pipeline, while everything else runs `default`.
## Example 3: default, push and web
Set `$NIX_CI_DEFAULT_SOURCES` to `^(push|web)$`.
## Example 4: always run the specific pipelines, never default
Set `$NIX_CI_DEFAULT_SOURCES` to any regex that never matches the sources,
like `a\A` or `nothing`.

75
docs/setup.md Normal file
View file

@ -0,0 +1,75 @@
# Setup
To integrate Nix GitLab CI into your project, you need to make two main changes:
1. Add the `nix-gitlab-ci` flake module to your `flake.nix`.
1. Include the necessary component in your `.gitlab-ci.yml`.
## Adding to `flake.nix`
In your project's `flake.nix`, add `nix-gitlab-ci` as an input and import its
flake module within your `flake-parts` configuration.
```nix title="flake.nix"
{
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; # Or your preferred nixpkgs branch/version
flake-parts.url = "github:hercules-ci/flake-parts";
# Add nix-gitlab-ci as an input
# recommendation: pin to a specific release/version
nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/<version>?dir=lib";
};
outputs = { nixpkgs, flake-parts, ... }@inputs:
flake-parts.lib.mkFlake { inherit inputs; } {
imports = [
# Import the nix-gitlab-ci flake module
inputs.nix-gitlab-ci.flakeModule
];
systems = [
"x86_64-linux"
"aarch64-linux"
# Add other systems you need
];
perSystem = { pkgs, ... }: {
# define your CI pipelines
# ci = { ... };
# pipelines."merge_request_event" = { ... };
};
};
}
```
Replace `<version>` with the specific version or commit hash of `nix-gitlab-ci`
you wish to use. Pinning to a specific version is highly recommended for
reproducibility and compatibility.
!!! warning
While the flake input is locked through `flake.lock`, the CI/CD component
will always use the latest commit of the reference. This means that by using
a branch like `main` as version for both, the CI/CD component will always use
the latest commit while your flake uses a fixed one.
This could result in drift between both, potentially breaking stuff.
## Including in `.gitlab-ci.yml`
Your `.gitlab-ci.yml` file will be minimal. Its primary role is to include the
`nix-gitlab-ci` component, which will then generate the full CI configuration
based on your Nix code.
```yaml title=".gitlab-ci.yml"
include:
- component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@<version>
inputs:
# This input sets the Docker image tag used for the CI jobs.
# Use the same version as you pinned in your flake.nix for consistency.
version: <version>
```
Again, ensure `<version>` matches the version used in your `flake.nix`.
This component includes a job (`build:nix-ci`) that will evaluate your Nix
configuration and generate the `.gitlab-ci.yml` used for the pipeline run.

33
docs/usage.md Normal file
View file

@ -0,0 +1,33 @@
# Usage
To create a basic pipeline, configure it by setting `ci` in `perSystem`.
The schema is similar to the `.gitlab-ci.yml`, only jobs are defined differently:
```nix
ci = {
# Nix GitLab CI specific config, see `configType` in `flakeModule.nix`
config = {};
jobs = {
"job-a" = {};
"job-b" = {};
};
};
```
For every job, there are a couple of settings you can adjust aswell:
```nix
"job-a" = {
# see `jobType` in `flakeModule.nix`
nix = {
enable = true; # is this a nix-based job?
deps = []; # dependencies to install for this job
# for gitlab runner cache:
enable-runner-cache = false;
runner-cache-key = "";
};
};
```
Since V2 multiple pipelines are supported.
See [Multiple Pipelines](./multi_pipeline.md) for more.

59
docs/utilities.md Normal file
View file

@ -0,0 +1,59 @@
# Utilities
Nix GitLab CI provides a couple of utilities to help with development and
debugging.
## Disabling caching temporarily
Nix GitLab CI often utilizes caching mechanisms to speed up your pipelines
(see [Caching](./caching.md)).
However, there might be situations where you need to temporarily disable these
caches for a specific pipeline run, for example, to debug a caching issue or
ensure a clean build.
To disable most of the provided caches for a pipeline, set the environment
variable `NIX_CI_DISABLE_CACHE` to any non-empty value (e.g., `yes`, `true`, `1`)
when triggering the pipeline in the GitLab UI or via the API.
## Forcing a rebuild of the CI pipeline definition
The job responsible for generating the `.gitlab-ci.yml` from your Nix code
(`build:nix-ci`) might itself be cached. If you've made changes to your Nix CI
configuration and the pipeline doesn't seem to pick them up, the cached job
definition might be the reason.
You should first double check if all the Nix files you defined the CI config in
are specified in the `cache_files` CI/CD-component input
(see [CI/CD Component](./cicd_component.md) for more).
To force this specific job to rebuild and re-evaluate your Nix configuration,
set the environment variable `NIX_CI_FORCE_BUILD` when triggering the pipeline.
## Running jobs locally
One of the benefits of defining your CI jobs with Nix is the ability to run them
locally in an environment that closely mirrors the CI environment. This can
significantly speed up debugging and development.
You can run the script of any defined job locally using the `nix run` command.
The syntax is:
```sh
nix run .#gitlab-ci:pipeline:<pipeline name>:job:<job name>
```
Replace `<pipeline name>` with the name of the pipeline the job belongs to
(e.g., `default` for jobs defined under the `ci` attribute) and `<job name>`
with the name of the job you want to run.
This command will set up the environment with the specified `nix.deps` and
execute the job's `script`.
There is also an attribute `.#gitlab-ci:pipeline:<pipeline name>:job-deps:<job name>`.
Building this derivation will generate a shell script which exports the required
environment variables for the job, such as the `PATH` including all dependencies
and any custom environment variables that contain store paths (ensuring they are
correctly resolved across different architectures).
You can use this to inspect the environment that would be set up for a job without
running the full script.

179
flake.lock generated
View file

@ -14,11 +14,11 @@
"nixpkgs": "nixpkgs" "nixpkgs": "nixpkgs"
}, },
"locked": { "locked": {
"lastModified": 1728672398, "lastModified": 1737621947,
"narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", "narHash": "sha256-8HFvG7fvIFbgtaYAY2628Tb89fA55nPm2jSiNs0/Cws=",
"owner": "cachix", "owner": "cachix",
"repo": "cachix", "repo": "cachix",
"rev": "aac51f698309fd0f381149214b7eee213c66ef0a", "rev": "f65a3cd5e339c223471e64c051434616e18cc4f5",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -32,18 +32,16 @@
"inputs": { "inputs": {
"cachix": "cachix", "cachix": "cachix",
"flake-compat": "flake-compat", "flake-compat": "flake-compat",
"git-hooks": [ "git-hooks": "git-hooks",
"git-hooks"
],
"nix": "nix", "nix": "nix",
"nixpkgs": "nixpkgs_3" "nixpkgs": "nixpkgs_3"
}, },
"locked": { "locked": {
"lastModified": 1732585607, "lastModified": 1743687534,
"narHash": "sha256-6ffeaSMuaL326f7KrCeScpSJtdHsFKS9gPrsSZkndvU=", "narHash": "sha256-QIh5jKWE0aZ4N77Zu3kz0RjA22zqqy5p6PfJnOW5rPE=",
"owner": "cachix", "owner": "cachix",
"repo": "devenv", "repo": "devenv",
"rev": "a520f05c40ebecaf5e17064b27e28ba8e70c49fb", "rev": "efd5d68a6483573410565d0b940e1a67b6f92591",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -55,27 +53,11 @@
"flake-compat": { "flake-compat": {
"flake": false, "flake": false,
"locked": { "locked": {
"lastModified": 1696426674, "lastModified": 1733328505,
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
"owner": "edolstra", "owner": "edolstra",
"repo": "flake-compat", "repo": "flake-compat",
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1696426674,
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -111,11 +93,11 @@
"nixpkgs-lib": "nixpkgs-lib" "nixpkgs-lib": "nixpkgs-lib"
}, },
"locked": { "locked": {
"lastModified": 1730504689, "lastModified": 1743550720,
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", "narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=",
"owner": "hercules-ci", "owner": "hercules-ci",
"repo": "flake-parts", "repo": "flake-parts",
"rev": "506278e768c2a08bec68eb62932193e341f55c90", "rev": "c621e8422220273271f52058f618c94e405bb0f5",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -126,17 +108,21 @@
}, },
"git-hooks": { "git-hooks": {
"inputs": { "inputs": {
"flake-compat": "flake-compat_2", "flake-compat": [
"devenv"
],
"gitignore": "gitignore", "gitignore": "gitignore",
"nixpkgs": "nixpkgs_4", "nixpkgs": [
"nixpkgs-stable": "nixpkgs-stable" "devenv",
"nixpkgs"
]
}, },
"locked": { "locked": {
"lastModified": 1732021966, "lastModified": 1740849354,
"narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", "narHash": "sha256-oy33+t09FraucSZ2rZ6qnD1Y1c8azKKmQuCvF2ytUko=",
"owner": "cachix", "owner": "cachix",
"repo": "git-hooks.nix", "repo": "git-hooks.nix",
"rev": "3308484d1a443fc5bc92012435d79e80458fe43c", "rev": "4a709a8ce9f8c08fa7ddb86761fe488ff7858a07",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -148,6 +134,7 @@
"gitignore": { "gitignore": {
"inputs": { "inputs": {
"nixpkgs": [ "nixpkgs": [
"devenv",
"git-hooks", "git-hooks",
"nixpkgs" "nixpkgs"
] ]
@ -182,6 +169,21 @@
"type": "github" "type": "github"
} }
}, },
"mkdocs-material-umami": {
"locked": {
"lastModified": 1745840856,
"narHash": "sha256-1Ad1JTMQMP6YsoIKAA+SBCE15qWrYkGue9/lXOLnu9I=",
"owner": "technofab",
"repo": "mkdocs-material-umami",
"rev": "3ac9b194450f6b779c37b8d16fec640198e5cd0a",
"type": "gitlab"
},
"original": {
"owner": "technofab",
"repo": "mkdocs-material-umami",
"type": "gitlab"
}
},
"nix": { "nix": {
"inputs": { "inputs": {
"flake-compat": [ "flake-compat": [
@ -201,11 +203,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1727438425, "lastModified": 1741798497,
"narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=", "narHash": "sha256-E3j+3MoY8Y96mG1dUIiLFm2tZmNbRvSiyN7CrSKuAVg=",
"owner": "domenkozar", "owner": "domenkozar",
"repo": "nix", "repo": "nix",
"rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546", "rev": "f3f44b2baaf6c4c6e179de8cbb1cc6db031083cd",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -215,13 +217,30 @@
"type": "github" "type": "github"
} }
}, },
"nix-mkdocs": {
"locked": {
"dir": "lib",
"lastModified": 1745175318,
"narHash": "sha256-eJOTw3SK4psqBPP2hNJ4D/13/oi/FLoM0tYKoCGVFP8=",
"owner": "technofab",
"repo": "nixmkdocs",
"rev": "6d6e0139060c896ae14de4b9c82335655a384643",
"type": "gitlab"
},
"original": {
"dir": "lib",
"owner": "technofab",
"repo": "nixmkdocs",
"type": "gitlab"
}
},
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1730531603, "lastModified": 1733212471,
"narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", "narHash": "sha256-M1+uCoV5igihRfcUKrr1riygbe73/dzNnzPsmaLCmpo=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", "rev": "55d15ad12a74eb7d4646254e13638ad0c4128776",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -233,29 +252,16 @@
}, },
"nixpkgs-lib": { "nixpkgs-lib": {
"locked": { "locked": {
"lastModified": 1730504152, "lastModified": 1743296961,
"narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=", "narHash": "sha256-b1EdN3cULCqtorQ4QeWgLMrd5ZGOjLSLemfa00heasc=",
"type": "tarball", "owner": "nix-community",
"url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" "repo": "nixpkgs.lib",
}, "rev": "e4822aea2a6d1cdd36653c134cacfd64c97ff4fa",
"original": {
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1730741070,
"narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d063c1dd113c91ab27959ba540c0d9753409edf3",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "NixOS", "owner": "nix-community",
"ref": "nixos-24.05", "repo": "nixpkgs.lib",
"repo": "nixpkgs",
"type": "github" "type": "github"
} }
}, },
@ -277,11 +283,11 @@
}, },
"nixpkgs_3": { "nixpkgs_3": {
"locked": { "locked": {
"lastModified": 1716977621, "lastModified": 1733477122,
"narHash": "sha256-Q1UQzYcMJH4RscmpTkjlgqQDX5yi1tZL0O345Ri6vXQ=", "narHash": "sha256-qamMCz5mNpQmgBwc8SB5tVMlD5sbwVIToVZtSxMph9s=",
"owner": "cachix", "owner": "cachix",
"repo": "devenv-nixpkgs", "repo": "devenv-nixpkgs",
"rev": "4267e705586473d3e5c8d50299e71503f16a6fb6", "rev": "7bd9e84d0452f6d2e63b6e6da29fe73fac951857",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -293,11 +299,11 @@
}, },
"nixpkgs_4": { "nixpkgs_4": {
"locked": { "locked": {
"lastModified": 1730768919, "lastModified": 1745377448,
"narHash": "sha256-8AKquNnnSaJRXZxc5YmF/WfmxiHX6MMZZasRP6RRQkE=", "narHash": "sha256-jhZDfXVKdD7TSEGgzFJQvEEZ2K65UMiqW5YJ2aIqxMA=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "a04d33c0c3f1a59a2c1cb0c6e34cd24500e5a1dc", "rev": "507b63021ada5fee621b6ca371c4fca9ca46f52c",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -309,27 +315,11 @@
}, },
"nixpkgs_5": { "nixpkgs_5": {
"locked": { "locked": {
"lastModified": 1732238832, "lastModified": 1735554305,
"narHash": "sha256-sQxuJm8rHY20xq6Ah+GwIUkF95tWjGRd1X8xF+Pkk38=", "narHash": "sha256-zExSA1i/b+1NMRhGGLtNfFGXgLtgo+dcuzHzaWA6w3Q=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "8edf06bea5bcbee082df1b7369ff973b91618b8d",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_6": {
"locked": {
"lastModified": 1731890469,
"narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=",
"owner": "nixos", "owner": "nixos",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "5083ec887760adfe12af64830a66807423a859a7", "rev": "0e82ab234249d8eee3e8c91437802b32c74bb3fd",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -343,8 +333,9 @@
"inputs": { "inputs": {
"devenv": "devenv", "devenv": "devenv",
"flake-parts": "flake-parts_2", "flake-parts": "flake-parts_2",
"git-hooks": "git-hooks", "mkdocs-material-umami": "mkdocs-material-umami",
"nixpkgs": "nixpkgs_5", "nix-mkdocs": "nix-mkdocs",
"nixpkgs": "nixpkgs_4",
"systems": "systems", "systems": "systems",
"treefmt-nix": "treefmt-nix" "treefmt-nix": "treefmt-nix"
} }
@ -366,14 +357,14 @@
}, },
"treefmt-nix": { "treefmt-nix": {
"inputs": { "inputs": {
"nixpkgs": "nixpkgs_6" "nixpkgs": "nixpkgs_5"
}, },
"locked": { "locked": {
"lastModified": 1732643199, "lastModified": 1743677901,
"narHash": "sha256-uI7TXEb231o8dkwB5AUCecx3AQtosRmL6hKgnckvjps=", "narHash": "sha256-eWZln+k+L/VHO69tUTzEmgeDWNQNKIpSUa9nqQgBrSE=",
"owner": "numtide", "owner": "numtide",
"repo": "treefmt-nix", "repo": "treefmt-nix",
"rev": "84637a7ab04179bdc42aa8fd0af1909fba76ad0c", "rev": "57dabe2a6255bd6165b2437ff6c2d1f6ee78421a",
"type": "github" "type": "github"
}, },
"original": { "original": {

319
flake.nix
View file

@ -8,14 +8,15 @@
imports = [ imports = [
inputs.devenv.flakeModule inputs.devenv.flakeModule
inputs.treefmt-nix.flakeModule inputs.treefmt-nix.flakeModule
inputs.nix-mkdocs.flakeModule
./lib/flakeModule.nix ./lib/flakeModule.nix
]; ];
systems = import systems; systems = import systems;
flake = {}; flake = {};
perSystem = { perSystem = {
pkgs, pkgs,
inputs',
config, config,
system,
... ...
}: rec { }: rec {
treefmt = { treefmt = {
@ -25,6 +26,15 @@
mdformat.enable = true; mdformat.enable = true;
yamlfmt.enable = true; yamlfmt.enable = true;
}; };
settings.formatter = {
yamlfmt.excludes = ["templates/nix-gitlab-ci.yml"];
mdformat.command = let
pkg = pkgs.python3.withPackages (p: [
p.mdformat
p.mdformat-mkdocs
]);
in "${pkg}/bin/mdformat";
};
}; };
devenv.shells.default = { devenv.shells.default = {
containers = pkgs.lib.mkForce {}; containers = pkgs.lib.mkForce {};
@ -39,16 +49,103 @@
}; };
}; };
}; };
doc = {
path = ./docs;
deps = pp: [
pp.mkdocs-material
(pp.callPackage inputs.mkdocs-material-umami {})
];
config = {
site_name = "Nix GitLab CI";
repo_name = "TECHNOFAB/nix-gitlab-ci";
repo_url = "https://gitlab.com/TECHNOFAB/nix-gitlab-ci";
edit_uri = "edit/main/docs/";
theme = {
name = "material";
features = ["content.code.copy" "content.action.edit"];
icon.repo = "simple/gitlab";
logo = "images/logo.png";
favicon = "images/favicon.png";
palette = [
{
scheme = "default";
media = "(prefers-color-scheme: light)";
primary = "deep orange";
accent = "orange";
toggle = {
icon = "material/brightness-7";
name = "Switch to dark mode";
};
}
{
scheme = "slate";
media = "(prefers-color-scheme: dark)";
primary = "deep orange";
accent = "orange";
toggle = {
icon = "material/brightness-4";
name = "Switch to light mode";
};
}
];
};
plugins = ["search" "material-umami"];
nav = [
{"Introduction" = "index.md";}
{"Setup" = "setup.md";}
{"Usage" = "usage.md";}
{"CI/CD Component" = "cicd_component.md";}
{"Environment Variables" = "environment_variables.md";}
{"Caching" = "caching.md";}
{"Multiple Pipelines" = "multi_pipeline.md";}
{"Utilities" = "utilities.md";}
{"Kubernetes Runner Example" = "kubernetes_runner.md";}
{"Example Configs" = "examples.md";}
];
markdown_extensions = [
{
"pymdownx.highlight".pygments_lang_class = true;
}
"pymdownx.inlinehilite"
"pymdownx.snippets"
"pymdownx.superfences"
"fenced_code"
"admonition"
];
extra.analytics = {
provider = "umami";
site_id = "28f7c904-db22-4c2b-9ee4-ed42e14b6db9";
src = "https://analytics.tf/umami";
domains = "nix-gitlab-ci.projects.tf";
feedback = {
title = "Was this page helpful?";
ratings = [
{
icon = "material/thumb-up-outline";
name = "This page is helpful";
data = "good";
note = "Thanks for your feedback!";
}
{
icon = "material/thumb-down-outline";
name = "This page could be improved";
data = "bad";
note = "Thanks for your feedback! Please leave feedback by creating an issue :)";
}
];
};
};
};
};
# should set the "default" pipeline
ci = { ci = {
# use the image built in the parent pipeline for dogfooding stages = ["test" "build" "deploy"];
config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA";
stages = ["test"];
jobs = { jobs = {
"test" = { "test" = {
stage = "test"; stage = "test";
nix = { nix = {
deps = [pkgs.hello pkgs.curl]; deps = [pkgs.hello pkgs.curl];
disable-cache = false; enable-runner-cache = true;
}; };
variables = { variables = {
TEST = "test"; TEST = "test";
@ -60,6 +157,11 @@
"echo $TEST $TEST_WITH_DERIVATION" "echo $TEST $TEST_WITH_DERIVATION"
]; ];
}; };
"test-default" = {
stage = "test";
nix.deps = [pkgs.hello];
script = ["hello"];
};
"test-non-nix" = { "test-non-nix" = {
nix.enable = false; nix.enable = false;
stage = "test"; stage = "test";
@ -68,132 +170,89 @@
"echo \"This job will not be modified to use nix\"" "echo \"This job will not be modified to use nix\""
]; ];
}; };
# -- actually useful jobs --
"docs" = {
stage = "build";
script = [
# sh
''
nix build .#docs:default
mkdir -p public
cp -r result/. public/
''
];
artifacts.paths = ["public"];
};
"pages" = {
nix.enable = false;
image = "alpine:latest";
stage = "deploy";
script = ["true"];
artifacts.paths = ["public"];
rules = [
{
"if" = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH";
}
];
};
};
};
pipelines."non-default" = {
stages = ["test"];
jobs = {
"test" = {
stage = "test";
script = [
"echo Hello from another pipeline"
];
};
}; };
}; };
packages = let packages = let
setupScript = extra_setup: setupScript = pkgs.writeShellScriptBin "setup_nix_ci" (builtins.readFile ./scripts/setup_nix_ci.sh);
pkgs.writeShellScriptBin "setup_nix_ci" '' finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" (builtins.readFile ./scripts/finalize_nix_ci.sh);
echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI"
nix path-info --all > /tmp/nix-store-before
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
${extra_setup}
else
echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration"
fi
export NIX_CONFIG="
extra-trusted-public-keys = $NIX_PUBLIC_KEYS
extra-trusted-substituters = $NIX_SUBSTITUTERS
extra-substituters = $NIX_SUBSTITUTERS
$NIX_CONFIG
$NIX_EXTRA_CONFIG
"
echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"
# load the job's deps only if the name was passed
if [[ ! -z $1 ]]; then
echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job"
nix build .#gitlab-ci-job-deps:$1
source $(readlink -f result)
echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K"
fi
'';
finalizeScript = push_command:
pkgs.writeShellScriptBin "finalize_nix_ci" ''
echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache"
nix path-info --all > /tmp/nix-store-after
${pkgs.diffutils}/bin/diff --new-line-format="%L" \
--old-line-format="" --unchanged-line-format="" \
/tmp/nix-store-before /tmp/nix-store-after \
| {
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
${push_command}
else
${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; }
fi
}
echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
'';
mkImage = extraPackages:
pkgs.dockerTools.buildImage {
name = "nix-gitlab-ci";
fromImage = pkgs.dockerTools.pullImage {
imageName = "nixpkgs/nix-flakes";
imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c";
sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf";
finalImageName = "nixpkgs/nix-flakes";
finalImageTag = "latest";
};
copyToRoot = pkgs.buildEnv {
name = "image-root";
paths =
[
pkgs.gitMinimal
pkgs.gnugrep
]
++ extraPackages;
pathsToLink = ["/bin"];
};
};
in { in {
setup-script = setup-script = setupScript;
setupScript finalize-script = finalizeScript;
# sh image = pkgs.dockerTools.buildImage {
'' name = "nix-ci";
# extra_setup fromImage = let
true hashes = {
''; "x86_64-linux" = "sha256-kJ7dqje5o1KPr3RDZ7/THbhMSoiCU1C/7HshDrNfwnM=";
finalize-script = "aarch64-linux" = "sha256-jz+Z3Ji+hy5d9ImOh/YOKCqy9P9/cseSov+5J/O95bg=";
finalizeScript };
# sh # check digest of tags like nixos-24.11-aarch64-linux etc.
'' digests = {
# push_command "x86_64-linux" = "sha256:345f210dea4cbd049e2d01d13159c829066dfb6e273cdd49ea878186d17b19f7";
true "aarch64-linux" = "sha256:66163fdf446d851416dd4e9be28c0794d9c2550214a57a846957699a3f5747f6";
''; };
image = mkImage [ hash = hashes.${system} or (throw "Unsupported system");
(setupScript imageDigest = digests.${system} or (throw "Unsupported system");
# sh in
'' pkgs.dockerTools.pullImage {
cachedir="$(pwd)/.nix-cache" imageName = "nixpkgs/nix-flakes";
echo "Configuring caching with the Runner Cache in $cachedir..." inherit hash imageDigest;
export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true" };
'') copyToRoot = pkgs.buildEnv {
(finalizeScript name = "image-root";
# sh paths = with pkgs;
'' [
# add ^* to all store paths ending in .drv (prevent warning log spam) gitMinimal
${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true gnugrep
'') gnused
]; coreutils
image-cachix = mkImage [ diffutils
(setupScript cachix
# sh attic-client
'' ]
echo "Configuring caching with cachix..." ++ [
${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true setupScript
'') finalizeScript
(finalizeScript ];
# sh pathsToLink = ["/bin"];
'' };
${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true };
'')
];
image-attic = mkImage [
(setupScript
# sh
''
echo "Configuring caching with attic..."
${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true
${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true
'')
(finalizeScript
# sh
''
${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true
'')
];
}; };
checks = packages; checks = packages;
@ -206,12 +265,10 @@
# flake & devenv related # flake & devenv related
flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.url = "github:hercules-ci/flake-parts";
systems.url = "github:nix-systems/default-linux"; systems.url = "github:nix-systems/default-linux";
devenv = { devenv.url = "github:cachix/devenv";
url = "github:cachix/devenv";
inputs.git-hooks.follows = "git-hooks";
};
git-hooks.url = "github:cachix/git-hooks.nix";
treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.url = "github:numtide/treefmt-nix";
nix-mkdocs.url = "gitlab:technofab/nixmkdocs?dir=lib";
mkdocs-material-umami.url = "gitlab:technofab/mkdocs-material-umami";
}; };
nixConfig = { nixConfig = {

33
gitlab-ci.yml
View file

@ -1,33 +0,0 @@
#
# NOTE: DEPRECATED: please switch to the CI/CD Component or include "templates/nix-gitlab-ci.yml"
#
variables:
# latest | latest-cachix | latest-attic etc.
NIX_CI_IMAGE_TAG: latest
stages:
- build
- trigger
nix-ci:build:
stage: build
image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${NIX_CI_IMAGE_TAG}
before_script:
- source setup_nix_ci
script:
# build the generated-gitlab-ci.yml
- nix build .#gitlab-ci-config
- install result generated-gitlab-ci.yml
after_script:
# upload to binary cache
- finalize_nix_ci
artifacts:
paths:
- generated-gitlab-ci.yml
nix-ci:trigger:
stage: trigger
needs:
- nix-ci:build
trigger:
include:
- artifact: generated-gitlab-ci.yml
job: nix-ci:build
strategy: depend

450
lib/flakeModule.nix
View file

@ -9,137 +9,159 @@
pkgs, pkgs,
... ...
}: let }: let
inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList;
cfg = config.ci.config; cfg = config.ci.config;
stdenvMinimal = pkgs.stdenvNoCC.override {
cc = null;
preHook = "";
allowedRequisites = null;
initialPath = [pkgs.coreutils pkgs.findutils];
shell = "/bin/sh";
extraNativeBuildInputs = [];
};
filterAttrsRec = pred: v: filterAttrsRec = pred: v:
if lib.isAttrs v if isAttrs v
then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v) then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v)
else v; else v;
subType = options: lib.types.submodule {inherit options;}; subType = options: types.submodule {inherit options;};
mkNullOption = type: mkNullOption = type:
lib.mkOption { mkOption {
default = null; default = null;
type = lib.types.nullOr type; type = types.nullOr type;
}; };
configType = with lib; configType = subType {
subType { nix-jobs-by-default = mkOption {
default-nix-image = mkOption { type = types.bool;
type = types.str; default = true;
default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest"; description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
description = "The image to use on nix jobs";
};
nix-jobs-per-default = mkOption {
type = types.bool;
default = true;
description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false";
};
disable-cache = mkOption {
type = types.bool;
default = false;
description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE";
};
cache-key = mkOption {
type = types.str;
default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
description = "Cache key to use for the nix cache";
};
}; };
jobType = with lib; };
subType { jobType = subType {
# nix ci opts # nix ci opts
nix = mkOption { nix = mkOption {
type = subType {
enable = mkOption {
type = types.bool;
default = cfg.nix-jobs-per-default;
description = "Handle this job as a nix job";
};
deps = mkOption {
type = types.listOf types.package;
default = [];
description = "Dependencies/packages to install for this job";
};
disable-cache = mkOption {
type = types.bool;
default = cfg.disable-cache;
description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE";
};
cache-key = mkOption {
type = types.str;
default = cfg.cache-key;
description = "Cache key to use for the nix cache";
};
};
default = {};
description = "Configure Nix Gitlab CI for each job individually";
};
# gitlab opts
script = mkOption {
type = types.listOf types.str;
default = [];
};
stage = mkOption {
type = types.str;
default = "test";
};
image = mkOption {
type = types.str;
default = cfg.default-nix-image;
};
after_script = mkNullOption (types.listOf types.str);
allow_failure = mkNullOption (types.either types.attrs types.bool);
artifacts = mkNullOption (types.attrs);
before_script = mkNullOption (types.listOf types.str);
cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
coverage = mkNullOption (types.str);
dependencies = mkNullOption (types.listOf types.str);
environment = mkNullOption (types.either types.attrs types.str);
extends = mkNullOption (types.str);
hooks = mkNullOption (types.attrs);
id_tokens = mkNullOption (types.attrs);
"inherit" = mkNullOption (types.attrs);
interruptible = mkNullOption (types.bool);
needs = mkNullOption (types.listOf (types.either types.str types.attrs));
publish = mkNullOption (types.str);
pages = mkNullOption (types.attrs);
parallel = mkNullOption (types.either types.int types.attrs);
release = mkNullOption (types.attrs);
retry = mkNullOption (types.either types.int types.attrs);
rules = mkNullOption (types.listOf types.attrs);
resource_group = mkNullOption (types.str);
secrets = mkNullOption (types.attrs);
services = mkNullOption (types.listOf types.attrs);
start_in = mkNullOption (types.str);
tags = mkNullOption (types.listOf types.str);
timeout = mkNullOption (types.str);
variables = mkNullOption (types.attrs);
when = mkNullOption (types.str);
};
in {
options = with lib; {
ci = mkOption {
type = subType { type = subType {
config = mkOption { enable = mkOption {
type = configType; type = types.bool;
description = '' default = cfg.nix-jobs-by-default;
Configuration options for the nix part itself description = "Handle this job as a nix job";
'';
default = {};
}; };
image = mkNullOption (types.str); deps = mkOption {
variables = mkNullOption (types.attrs); type = types.listOf types.package;
default = mkNullOption (types.attrs); default = [];
stages = mkNullOption (types.listOf types.str); description = "Dependencies/packages to install for this job";
include = mkNullOption (types.attrs); };
workflow = mkNullOption (types.attrs); enable-runner-cache = mkOption {
jobs = mkOption { type = types.bool;
type = types.lazyAttrsOf jobType; default = false;
default = {}; description = ''
Cache this job using the GitLab Runner cache.
Warning: useful for tiny jobs, but most of the time it just takes an eternity.
'';
};
runner-cache-key = mkOption {
type = types.str;
default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG";
description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true";
}; };
}; };
default = {};
description = "Configure Nix Gitlab CI for each job individually";
};
# gitlab opts
script = mkOption {
type = types.listOf types.str;
default = [];
};
stage = mkOption {
type = types.str;
default = "test";
};
image = mkOption {
type = types.str;
default = "$NIX_CI_IMAGE";
};
after_script = mkNullOption (types.listOf types.str);
allow_failure = mkNullOption (types.either types.attrs types.bool);
artifacts = mkNullOption (types.attrs);
before_script = mkNullOption (types.listOf types.str);
cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs);
coverage = mkNullOption (types.str);
dependencies = mkNullOption (types.listOf types.str);
environment = mkNullOption (types.either types.attrs types.str);
extends = mkNullOption (types.str);
hooks = mkNullOption (types.attrs);
id_tokens = mkNullOption (types.attrs);
"inherit" = mkNullOption (types.attrs);
interruptible = mkNullOption (types.bool);
needs = mkNullOption (types.listOf (types.either types.str types.attrs));
publish = mkNullOption (types.str);
pages = mkNullOption (types.attrs);
parallel = mkNullOption (types.either types.int types.attrs);
release = mkNullOption (types.attrs);
retry = mkNullOption (types.either types.int types.attrs);
rules = mkNullOption (types.listOf types.attrs);
resource_group = mkNullOption (types.str);
secrets = mkNullOption (types.attrs);
services = mkNullOption (types.listOf types.attrs);
start_in = mkNullOption (types.str);
tags = mkNullOption (types.listOf types.str);
timeout = mkNullOption (types.str);
variables = mkNullOption (types.attrs);
when = mkNullOption (types.str);
};
ciType = subType {
config = mkOption {
type = configType;
description = '' description = ''
Configuration options for the nix part itself
'';
default = {};
};
image = mkNullOption (types.str);
variables = mkNullOption (types.attrs);
default = mkNullOption (types.attrs);
stages = mkNullOption (types.listOf types.str);
include = mkNullOption (types.attrs);
workflow = mkNullOption (types.attrs);
jobs = mkOption {
type = types.lazyAttrsOf jobType;
default = {};
};
};
in {
options = {
pipelines = mkOption {
type = types.lazyAttrsOf ciType;
description = ''
Create multiple GitLab CI pipelines.
See README.md for more information about how a pipeline is selected.
'';
default = {};
apply = op: let
# NOTE: show warning if "default" is set and config.ci is not {}
legacyMode = config.ci != {};
defaultExists = builtins.hasAttr "default" op;
value =
{
"default" = config.ci;
}
// op;
in
if defaultExists && legacyMode
then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value
else value;
};
ci = mkOption {
type = ciType;
description = ''
Note: this is a shorthand for writing `pipelines."default"`
Generate a Gitlab CI configuration which can be used to trigger a child pipeline. Generate a Gitlab CI configuration which can be used to trigger a child pipeline.
This will inject code which pre-downloads the nix deps before each job and adds them to PATH. This will inject code which pre-downloads the nix deps before each job and adds them to PATH.
''; '';
@ -148,7 +170,10 @@
}; };
config.legacyPackages = let config.legacyPackages = let
toYaml = (pkgs.formats.yaml {}).generate; # NOTE: json is also valid yaml and this removes dependency on jq
# and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml
# respectively)
toYaml = name: value: builtins.toFile name (builtins.toJSON value);
mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set));
prepend = key: arr: job: prepend = key: arr: job:
job job
@ -175,94 +200,111 @@
} }
) )
(job.variables or {}); (job.variables or {});
jobs = filterAttrsRec (n: v: v != null) config.ci.jobs;
rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]);
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
# pro: we don't have to download everything, just the deps for the current job
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
jobsMappedForDeps =
mapAttrs (key: job: let
variablesWithStorePaths = filterJobVariables true job;
variableExports = lib.concatLines (
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
);
in {
name = "gitlab-ci-job-deps:${key}";
value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" ''
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
${variableExports}
'';
})
jobs;
# allows the user to directly run the script
jobsMappedForScript =
mapAttrs (key: job: let
variablesWithStorePaths = filterJobVariables false job;
variableExports = lib.concatLines (
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
);
in {
name = "gitlab-ci-job:${key}";
value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
# set up deps and environment variables containing store paths
. ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"}
# normal environment variables
${variableExports}
# run before_script, script and after_script
echo -e "\e[32mRunning before_script...\e[0m"
${lib.concatLines (job.before_script or [])}
echo -e "\e[32mRunning script...\e[0m"
${lib.concatLines job.script}
echo -e "\e[32mRunning after_script...\e[0m"
${lib.concatLines (job.after_script or [])}
'';
})
jobs;
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
jobsPatched =
mapAttrs (key: job: {
name = key;
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
builtins.removeAttrs (
(prependToBeforeScript [
"source setup_nix_ci ${key}"
]
(appendToAfterScript [
"finalize_nix_ci"
]
job))
// lib.optionalAttrs job.nix.enable {
image = job.image;
variables =
(filterJobVariables false job)
// lib.optionalAttrs job.nix.disable-cache {
NIX_CI_DISABLE_CACHE = "yes";
};
cache =
(
let
c = job.cache or [];
in
if builtins.isList c
then c
else [c]
)
++ (lib.optional (!job.nix.disable-cache) {
key = job.nix.cache-key;
paths = [".nix-cache/"];
});
}
) ["nix"];
})
jobs;
in in
{ lib.fold (pipeline: acc: acc // pipeline) {} (map (
gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); pipeline_name: let
} pipeline = config.pipelines."${pipeline_name}";
// jobsMappedForDeps jobs = filterAttrsRec (n: v: v != null) pipeline.jobs;
// jobsMappedForScript; rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]);
# this allows us to nix build this to get all the mentioned dependencies from the binary cache
# pro: we don't have to download everything, just the deps for the current job
# before, we just allowed pkgs inside the script string directly, but now with the ability to source this file
# we can support different architectures between runners (eg. the arch of the initial runner does not matter)
jobsMappedForDeps =
mapAttrs (key: job: let
variablesWithStorePaths = filterJobVariables true job;
variableExports = lib.concatLines (
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths
);
in {
name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}";
value = stdenvMinimal.mkDerivation {
name = "gitlab-ci-job-deps-${key}";
dontUnpack = true;
installPhase = let
script = ''
export PATH="${lib.makeBinPath job.nix.deps}:$PATH";
# variables containing nix derivations:
${variableExports}
'';
in
# sh
''
echo '${script}' > $out
chmod +x $out
'';
};
})
jobs;
# allows the user to directly run the script
jobsMappedForScript =
mapAttrs (key: job: let
variablesWithoutStorePaths = filterJobVariables false job;
variableExports = lib.concatLines (
lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths
);
in {
name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}";
value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" ''
# set up deps and environment variables containing store paths
. ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"}
# normal environment variables
${variableExports}
# run before_script, script and after_script
echo -e "\e[32mRunning before_script...\e[0m"
${lib.concatLines (job.before_script or [])}
echo -e "\e[32mRunning script...\e[0m"
${lib.concatLines job.script}
echo -e "\e[32mRunning after_script...\e[0m"
${lib.concatLines (job.after_script or [])}
'';
})
jobs;
# build the deps specific for this job before anything, this way the deps should be fetched from the cache
jobsPatched =
mapAttrs (key: job: {
name = key;
value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist";
builtins.removeAttrs (
(prependToBeforeScript [
"source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\""
]
(appendToAfterScript [
"finalize_nix_ci"
]
job))
// lib.optionalAttrs job.nix.enable {
image = job.image;
variables =
(filterJobVariables false job)
// lib.optionalAttrs job.nix.enable-runner-cache {
NIX_CI_CACHE_STRATEGY = "runner";
};
cache =
(
let
c = job.cache or [];
in
toList c
)
++ (lib.optional (job.nix.enable-runner-cache) {
key = job.nix.runner-cache-key;
paths = [".nix-cache/"];
});
}
) ["nix"];
})
jobs;
in
# gitlab-ci:pipeline:<name>
# gitlab-ci:pipeline:<name>:job:<name>
# gitlab-ci:pipeline:<name>:job-deps:<name>
{
"gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched);
}
// jobsMappedForDeps
// jobsMappedForScript
) (builtins.attrNames config.pipelines));
} }
); );
} }

41
scripts/finalize_nix_ci.sh Normal file
View file

@ -0,0 +1,41 @@
echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci[collapsed=true]\\r\\e[0KFinalizing Nix CI..."
nix path-info --all > /tmp/nix-store-after
echo "Finding new paths..."
NEW_PATHS=$(diff --new-line-format="%L" \
--old-line-format="" --unchanged-line-format="" \
/tmp/nix-store-before /tmp/nix-store-after)
COUNT=$(wc -l <<<"$NEW_PATHS")
if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then
export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}";
fi
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($NIX_CI_CACHE_STRATEGY)"
echo -n "$NEW_PATHS" | {
case "$NIX_CI_CACHE_STRATEGY" in
"runner")
export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"}
# add ^* to all store paths ending in .drv (prevent warning log spam)
sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true
;;
"attic")
attic push --stdin ci:$ATTIC_CACHE || true
;;
"cachix")
cachix push $CACHIX_CACHE || true
;;
"none")
echo "Cache strategy is none, doing nothing..."
;;
*)
echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'"
;;
esac
}
echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K"
else
echo "Caching disabled, not uploading $COUNT new store entries..."
fi
echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K"

49
scripts/setup_nix_ci.sh Normal file
View file

@ -0,0 +1,49 @@
echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI"
nix path-info --all > /tmp/nix-store-before
if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then
export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}";
echo "NIX_CI_CACHE_STRATEGY was set to auto, selected '$NIX_CI_CACHE_STRATEGY' for this job"
fi
if [ -z "$NIX_CI_DISABLE_CACHE" ]; then
echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($NIX_CI_CACHE_STRATEGY)"
case "$NIX_CI_CACHE_STRATEGY" in
"runner")
export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"}
echo "Runner Cache: $RUNNER_CACHE"
export NIX_CONFIG="$NIX_CONFIG
extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true
extra-substituters = $RUNNER_CACHE?priority=10&trusted=true
"
;;
"attic")
echo "Attic Cache: $ATTIC_CACHE"
attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true
attic use "$ATTIC_CACHE" || true
;;
"cachix")
echo "Cachix Cache: $CACHIX_CACHE"
cachix use "$CACHIX_CACHE" || true
;;
"none")
echo "Cache strategy is none, doing nothing..."
;;
*)
echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'"
;;
esac
echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K"
else
echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..."
fi
# load the job's deps only if the name was passed
if [[ ! -z $1 ]]; then
echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job"
nix build .#$1
source $(readlink -f result)
echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K"
fi
echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K"

86
templates/nix-gitlab-ci.yml
View file

@ -1,63 +1,103 @@
spec: spec:
inputs: inputs:
image_tag: cache_strategy:
type: string type: string
description: "latest | latest-cachix | latest-attic etc." description: |
default: latest auto (default) | none | runner | cachix | attic
Sets the default caching strategy.
- "auto": dynamically selects the best strategy for every job based on env variables
- "none": disables caching
- "runner", "cachix" & "attic": forces every job to use this strategy
Can be overridden by setting NIX_CI_CACHE_STRATEGY in the pipeline variables.
default: "auto"
cache_files: cache_files:
type: array type: array
description: | description: |
Files to use as the cache key for the generated pipeline yaml. Files to use as the cache key for the generated pipeline yaml.
If you use "ci.nix" to define CI, add that here for example If you use "ci.nix" to define CI, add that here for example.
Note that max 2 items are allowed in cache:key:files, so use something like
["flake.*", "ci.nix"] f. ex. to match flake.lock, flake.nix and ci.nix.
default: ["flake.nix", "flake.lock"] default: ["flake.nix", "flake.lock"]
disable_cache: version:
type: string type: string
description: | description: |
Disables any caching provided by this component. Set to any non-empty value to disable caching. Which version of the Nix CI image to use. Using a tag/version is recommended.
default: ""
--- ---
stages: stages:
- build - build
- trigger - trigger
variables: variables:
NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]${NIX_CI_DISABLE_CACHE:-}" # These can be overriden, see https://docs.gitlab.com/ci/variables/#cicd-variable-precedence
# which image should be used by default.
NIX_CI_IMAGE: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.version ]]
# default cache stategy
NIX_CI_CACHE_STRATEGY: $[[ inputs.cache_strategy ]]
nix-ci:build: nix-ci:build:
stage: build stage: build
image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]] image: $NIX_CI_IMAGE
cache: cache:
- key: - key:
files: $[[ inputs.cache_files ]] files: $[[ inputs.cache_files ]]
paths: paths:
- generated-gitlab-ci.yml - .nix-ci-pipelines/
- key: nix - key: nix
paths: paths:
- .nix-cache/ - .nix-cache/
before_script: before_script:
# generated-gitlab-ci.yml exists in the cache - |
- '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true' # if no explicit pipeline is requested
# allow the user to manually skip the cache (when the key files are not correctly configured etc.) if [[ -z "${NIX_CI_PIPELINE_NAME:-}" ]]; then
- '[ -n "$NIX_CI_FORCE_BUILD" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true' # if regex matches, use pipeline "default", otherwise $CI_PIPELINE_SOURCE
# only setup when we need to generate the pipeline yaml [[ "${CI_PIPELINE_SOURCE}" =~ ${NIX_CI_DEFAULT_SOURCES:-.*} ]] \
- 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi' && NIX_CI_PIPELINE_NAME="default" \
|| NIX_CI_PIPELINE_NAME="$CI_PIPELINE_SOURCE";
fi
echo "NIX_CI_GENERATED_PIPELINE_NAME=$NIX_CI_PIPELINE_NAME" >> trigger.env
# inheritance of pipeline variables is a bit weird, so explicitly override them
# (ctx: setting any of these in the project variables would only apply correctly
# in this pipeline, not the child pipeline, instead weirdly enough the default
# variables above are used). If any other variables are added at the top, add them
# here aswell
echo "NIX_CI_IMAGE=$NIX_CI_IMAGE" >> trigger.env
echo "NIX_CI_CACHE_STRATEGY=$NIX_CI_CACHE_STRATEGY" >> trigger.env
mkdir -p .nix-ci-pipelines/
# generated-gitlab-ci.yml exists in the cache
[[ -f ".nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml" ]] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true
# allow the user to manually skip the cache (when the key files are not correctly configured etc.)
[[ -n "$NIX_CI_FORCE_BUILD" ]] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true
# only setup when we need to generate the pipeline yaml
if [[ -z "$CACHED" ]]; then
source setup_nix_ci;
fi
script: script:
# build the generated-gitlab-ci.yml if it does not exist in the cache # build the pipeline if it does not exist in the cache
- 'if [ -z "$CACHED" ]; then nix build .#gitlab-ci-config && install result generated-gitlab-ci.yml; fi' - >
if [[ -z "$CACHED" ]]; then
nix build .#gitlab-ci:pipeline:${NIX_CI_PIPELINE_NAME} && install result .nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml;
fi
after_script: after_script:
# NOTE: environment variables of before_script and script don't exist here anymore
#
# save to binary cache or Gitlab CI cache only if we actually built something # save to binary cache or Gitlab CI cache only if we actually built something
# check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called # check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called
- 'if [ -f "/tmp/nix-store-before" ]; then finalize_nix_ci; fi' - |
if [[ -f "/tmp/nix-store-before" ]]; then
finalize_nix_ci;
fi
artifacts: artifacts:
paths: paths:
- generated-gitlab-ci.yml - .nix-ci-pipelines/
reports:
dotenv: trigger.env
nix-ci:trigger: nix-ci:trigger:
stage: trigger stage: trigger
needs: needs:
- nix-ci:build - nix-ci:build
trigger: trigger:
include: include:
- artifact: generated-gitlab-ci.yml - artifact: .nix-ci-pipelines/${NIX_CI_GENERATED_PIPELINE_NAME}.yml
job: nix-ci:build job: nix-ci:build
strategy: depend strategy: depend
forward: forward: