diff --git a/.envrc b/.envrc index a2d9328..f990172 100644 --- a/.envrc +++ b/.envrc @@ -1,4 +1 @@ -if ! use flake . --impure -then - echo "devenv could not be build. The devenv environment was not loaded. Make the necessary changes to flake.nix and hit enter to try again." >&2 -fi +use flake . --impure --accept-flake-config diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0393fb..24b9507 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ include: - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHA inputs: - image_tag: $CI_COMMIT_SHORT_SHA + version: $CI_COMMIT_SHORT_SHA stages: - build-images - build @@ -10,26 +10,40 @@ build:image: stage: build-images parallel: matrix: - - VARIANT: ["", "-cachix", "-attic"] + - ARCH: ["x86_64-linux", "aarch64-linux"] image: nixpkgs/nix-flakes:latest - before_script: - - nix profile install nixpkgs#skopeo - - export PATH="$PATH:$HOME/.nix-profile/bin" script: - - nix build .#image${VARIANT} + - nix build .#image --system $ARCH + after_script: + - install -D result dist/nix-ci-$ARCH.tar.gz + artifacts: + paths: + - dist +deploy:image: + stage: build-images + image: nixpkgs/nix-flakes:latest + needs: + - build:image + before_script: + - nix profile install nixpkgs#buildah + - export PATH="$PATH:$HOME/.nix-profile/bin" + - export REGISTRY_AUTH_FILE=''${HOME}/auth.json + - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + - mkdir -p /etc/containers && echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json + - mkdir -p /var/tmp + script: - export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-} - - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_SHORT_SHA}${VARIANT}" + - buildah manifest create localhost/nix-ci + - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz + - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz + - buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_SHORT_SHA} # branches - | if [ -z "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${NORMALIZED_BRANCH/main/latest}${VARIANT}"; + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${NORMALIZED_BRANCH/main/latest} fi # tags - | if [ -n "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_TAG}${VARIANT}"; + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_TAG} fi diff --git a/README.md b/README.md index 9735feb..ceaca42 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -# Nix Gitlab CI +# Nix GitLab CI + +[![built with nix](https://img.shields.io/static/v1?logo=nixos&logoColor=white&label=&message=Built%20with%20Nix&color=41439a)](https://builtwithnix.org) +[![pipeline status](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/badges/main/pipeline.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/commits/main) +![License: MIT](https://img.shields.io/gitlab/license/technofab/nix-gitlab-ci) +[![Latest Release](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/badges/release.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/releases) +[![Support me](https://img.shields.io/badge/Support-me-orange)](https://tec.tf/#support) +[![Docs](https://img.shields.io/badge/Read-Docs-orange)](https://nix-gitlab-ci.projects.tf) Flake module which allows generating a `.gitlab-ci.yml` from Nix. @@ -12,7 +19,7 @@ Also makes it possible to split CI parts in a separate module which can be impor # flake.nix { ... - inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; + inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; # recommendation: pin to the latest release/version outputs = {...}: flake-parts.lib.mkFlake {...} { imports = [ @@ -21,6 +28,7 @@ Also makes it possible to split CI parts in a separate module which can be impor ... perSystem = {pkgs, ...}: { + # ci is a shortcut and creates a "default" pipeline ci = { stages = ["test"]; jobs = { @@ -33,6 +41,11 @@ Also makes it possible to split CI parts in a separate module which can be impor }; }; }; + # runs on a merge request for example + pipelines."merge_request_event" = { + stages = ["some_stage"]; + jobs = { ... }; + }; ... } } @@ -42,10 +55,9 @@ Also makes it possible to split CI parts in a separate module which can be impor ```yaml # .gitlab-ci.yml include: - - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: use the latest version (try not to use latest) + - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: pin to the latest release/version (don't use "main" etc.) inputs: - # specify inputs here, for example: - image_tag: latest-cachix + version: # docker image tag, use the same version as a above ``` ## Utilities @@ -63,16 +75,15 @@ The `build:nix-ci` job has a different special environment variable `NIX_CI_FORC You can run any job's script (+ before and after) locally with Nix for easier testing: ```sh -nix run .#gitlab-ci-job: +# / pipeline name, like "default" +nix run .#gitlab-ci:pipeline::job: ``` -There is also `.#gitlab-ci-job-deps:` which generates and exports the required environment variables for each job: +There is also `.#gitlab-ci:pipeline::job-deps:` which generates and exports the required environment variables for each job: - PATH (with all deps) - any custom env variables which contain store paths to not break stuff when switching archs -Please see #8 for some issues and further improvements on this. - ## Thanks to Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix diff --git a/docs/caching.md b/docs/caching.md new file mode 100644 index 0000000..0a84110 --- /dev/null +++ b/docs/caching.md @@ -0,0 +1,50 @@ +# Caching + +Nix GitLab CI supports several caching mechanisms to speed up your pipelines. + +## GitLab Runner Cache + +The runner cache strategy copies the new store paths into a directory `.nix-cache`, +which is then saved in the regular GitLab cache (technically runner cache). +It's also configured as a substituter automatically. + +To enable, set the cache strategy to `runner`. + +Configure it using these environment variables: + +- `RUNNER_CACHE`: path to the runner cache (default `.nix-cache`) + +!!! warning + + This is very inefficient and should probably only be used for very very small + dependency counts. Otherwise it takes an eternity to save to cache. + +## Cachix + +Cachix is a hosted binary cache service that can significantly speed up Nix +builds by sharing build results. + +To enable, set the cache strategy to `attic`. + +Configure it using these environment variables: + +- `CACHIX_CACHE`: name of the cache to use +- (`CACHIX_AUTH_TOKEN`): cachix client itself uses this for authentication + +!!! warning + + Cachix has not been tested. Feedback is appreciated :) + +## Attic (Self-Hosted Cache) + +Attic is a self-hosted, deduplicating binary cache. It's a great option if you +want more control over your caching infrastructure and to have the cache closer +to your runners. + +To enable, set the cache strategy to `attic`. + +Configure it using these environment variables: + +- `ATTIC_SERVER`: URL of the server +- `ATTIC_CACHE`: name of the cache to use +- `ATTIC_TOKEN`: auth token from the attic server diff --git a/docs/cicd_component.md b/docs/cicd_component.md new file mode 100644 index 0000000..2fe2ae2 --- /dev/null +++ b/docs/cicd_component.md @@ -0,0 +1,41 @@ +# CI/CD Component + +The CI/CD Component has some inputs which configure defaults for Nix GitLab CI. + +## `version` + +- Type: `string` + +Which version of the Nix CI image to use. Using a tag/version is recommended. +Will not do anything if a custom image is specified using `NIX_CI_IMAGE`. + +## `cache_strategy` + +- Type: `string` +- Default: `"auto"` +- Options: `auto` | `none` | `runner` | `cachix` | `attic` + +Sets the default caching strategy. + +- `auto`: dynamically selects the best strategy for every job based on env variables +- `none`: disables caching +- `runner`, `cachix` & `attic`: forces every job to use this strategy + +Can be overridden by `NIX_CI_CACHE_STRATEGY`, see [Environment Variables](./environment_variables.md). + +## `cache_files` + +- Type: `array` (of strings) +- Default: `["flake.nix", "flake.lock"]` + +Files to use as the cache key for the generated pipeline yaml. +If you use a file like `ci.nix` to define CI, add that here for example. +This makes sure that changes to your Nix CI configuration will invalidate the cache, +otherwise an old pipeline yaml might be used. + +!!! warning + + The value of this is used in `cache:key:files`, which currently only supports + a max of 2 entries. So use something like `["flake.*", "ci.nix"]` to match + `flake.lock`, `flake.nix` and `ci.nix`. + See [gitlab-org/gitlab#301161](https://gitlab.com/gitlab-org/gitlab/-/issues/301161) diff --git a/docs/environment_variables.md b/docs/environment_variables.md new file mode 100644 index 0000000..56c0ba5 --- /dev/null +++ b/docs/environment_variables.md @@ -0,0 +1,107 @@ +# Environment Variables + +Nix GitLab CI is mostly controlled using environment variables. +This page outlines all the variables and their use case. + +## `NIX_CI_IMAGE` + +| | | +| ----------- | -------------------------------------------------------------------------- | +| Default | `registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci@$[[ inputs.version ]]` | +| Description | Image to use for the jobs | + +## `NIX_CI_PIPELINE_NAME` + +| | | +| ----------- | ------------------------------------------------- | +| Default | N/A | +| Description | Explicitly request a pipeline to be built and ran | +| See also | [Multi Pipeline](./multi_pipeline.md) | + +## `NIX_CI_DEFAULT_SOURCES` + +| | | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| Default | `.*` | +| Description | Regex to match `$CI_PIPELINE_SOURCE` against. If it matches, the `default` pipeline will be ran, otherwise `$CI_PIPELINE_SOURCE` | +| See also | [Multi Pipeline](./multi_pipeline.md) | + +## `NIX_CI_FORCE_BUILD` + +| | | +| ----------- | -------------------------------------------------------------------------------------- | +| Default | N/A | +| Description | Set to any non-empty value to force the `nix-ci:build` job to freshly build the config | +| See also | [Caching](./caching.md) | + +## `NIX_CI_DISABLE_CACHE` + +| | | +| ----------- | ------------------------------------------------------ | +| Default | N/A | +| Description | Set to any non-empty value to disable caching for jobs | +| See also | [Caching](./caching.md) | + +## `NIX_CI_CACHE_STRATEGY` + +| | | +| ----------- | --------------------------------------------------------------------------------- | +| Default | `$[[ inputs.cache_strategy ]]` -> defaults to `auto` | +| Description | Caching strategy to use. `auto` will select the strategy based on runner settings | +| See also | [Caching](./caching.md) | + +## `NIX_CI_RUNNER_CACHE_STRATEGY` + +| | | +| ----------- | ---------------------------------------------------------------- | +| Default | N/A | +| Description | Every runner can set it's own preferred cache strategy with this | +| See also | [Caching](./caching.md) | + +## `NIX_CI_DEFAULT_CACHE_STRATEGY` + +| | | +| ----------- | ------------------------------------------------------------------------------------------------- | +| Default | `none` | +| Description | If no runner cache strategy is set and the main strategy is set to auto, this will be the default | +| See also | [Caching](./caching.md) | + +## `RUNNER_CACHE` + +| | | +| ----------- | -------------------------------------- | +| Default | `.nix-cache` | +| Description | Path to directory for the runner cache | +| See also | [Caching](./caching.md) | + +## `CACHIX_CACHE` + +| | | +| ----------- | ------------------------------- | +| Default | N/A | +| Description | Name of the cachix cache to use | +| See also | [Caching](./caching.md) | + +## `ATTIC_CACHE` + +| | | +| ----------- | ------------------------------ | +| Default | N/A | +| Description | Name of the attic cache to use | +| See also | [Caching](./caching.md) | + +## `ATTIC_SERVER` + +| | | +| ----------- | ----------------------- | +| Default | N/A | +| Description | URL of the attic server | +| See also | [Caching](./caching.md) | + +## `ATTIC_TOKEN` + +| | | +| ----------- | ------------------------------- | +| Default | N/A | +| Description | API token from the attic server | +| See also | [Caching](./caching.md) | diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..6f82a63 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,18 @@ +# Example Configs + +## V2 + +- [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci) + See `flake.nix` for some random example jobs. +- [TECHNOFAB/nixlets](https://gitlab.com/TECHNOFAB/nixlets) +- [TECHNOFAB/nixmkdocs](https://gitlab.com/TECHNOFAB/nixmkdocs) +- [TECHNOFAB/tofunix](https://gitlab.com/TECHNOFAB/tofunix) + +## Old / V1 + +- [TECHNOFAB/coder-templates](https://gitlab.com/TECHNOFAB/coder-templates) + +!!! note + + Feel free to edit this page and add your project if you're using + Nix GitLab CI :) diff --git a/docs/images/favicon.png b/docs/images/favicon.png new file mode 100644 index 0000000..135b030 Binary files /dev/null and b/docs/images/favicon.png differ diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100644 index 0000000..2a70bf6 Binary files /dev/null and b/docs/images/logo.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..214fbf3 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,11 @@ +# Nix GitLab CI + +This project provides a Nix flake module that allows you to generate your `.gitlab-ci.yml` file directly from your Nix configuration. + +## Features + +- **Reproducibility:** Leverage Nix's strength in creating reproducible environments for your CI jobs. +- **Easy Dependency Management:** Easily include any package available in Nixpkgs or your own defined packages within your CI jobs using Nix. +- **Modularity:** Define and manage your CI configurations in a structured and modular way using Nix modules, making it easier to share and reuse CI logic across multiple projects. + +This documentation will guide you through setting up and using Nix GitLab CI for your projects. diff --git a/docs/kubernetes_runner.md b/docs/kubernetes_runner.md new file mode 100644 index 0000000..4c20421 --- /dev/null +++ b/docs/kubernetes_runner.md @@ -0,0 +1,38 @@ +# Kubernetes Runner Setup + +Using the GitLab Kubernetes runner allows your CI jobs to run as pods in a Kubernetes cluster. +Nix GitLab CI can be integrated with this setup, and using advanced configuration options like +`pod_spec` makes it easy to add runner specific caching. + +Using this Runner configuration ... + +```toml +[[runners.kubernetes.pod_spec]] +name = "nix-ci-cache-secrets" +patch = ''' + containers: + - name: build + envFrom: + - secretRef: + name: nix-ci-cache-env +''' +``` + +... and a secret containing ... + +```yaml +NIX_CI_RUNNER_CACHE_STRATEGY: attic +ATTIC_SERVER: # example: http://atticd..svc.cluster.local:8080 +ATTIC_CACHE: ci # name however you want, just needs to exist +ATTIC_TOKEN: +``` + +... makes your jobs automatically cache their Nix store paths to the in-cluster +attic when running with this runner. + +Other runners could use cachix or no cache, you get the idea ;P + +!!! note + + This of course works with any executor where you can set environment + variables. This is just an example how to do it in Kubernetes easily. diff --git a/docs/multi_pipeline.md b/docs/multi_pipeline.md new file mode 100644 index 0000000..d8dbfe6 --- /dev/null +++ b/docs/multi_pipeline.md @@ -0,0 +1,31 @@ +# Multiple Pipelines + +With V2, Nix GitLab CI can generate different pipelines, depending on the +pipeline source (`$CI_PIPELINE_SOURCE`). + +By default, no matter which source, the `default` pipeline is built and ran. +`$NIX_CI_PIPELINE` can override that, eg. when manually triggering a run. +To configure which source should be 1-to-1 translated to a pipeline with the +same name, set `$NIX_CI_DEFAULT_SOURCES` to a regex which explicitly does not +match these sources. Or set it to an impossible to match regex, then it will +always run the pipeline named after `$CI_PIPELINE_SOURCE`. + +## Example 1: always run default + +If you only have a single pipeline, you just have to call it `default`. +Everything else works out of the box. + +## Example 2: default and merge_request_event + +If you want the source `merge_request_event` to trigger a different pipeline, +name it like that and set `$NIX_CI_DEFAULT_SOURCES` to `^(merge_request_event)$`. +Now a merge request will run this pipeline, while everything else runs `default`. + +## Example 3: default, push and web + +Set `$NIX_CI_DEFAULT_SOURCES` to `^(push|web)$`. + +## Example 4: always run the specific pipelines, never default + +Set `$NIX_CI_DEFAULT_SOURCES` to any regex that never matches the sources, +like `a\A` or `nothing`. diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..d3aa96e --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,75 @@ +# Setup + +To integrate Nix GitLab CI into your project, you need to make two main changes: + +1. Add the `nix-gitlab-ci` flake module to your `flake.nix`. +1. Include the necessary component in your `.gitlab-ci.yml`. + +## Adding to `flake.nix` + +In your project's `flake.nix`, add `nix-gitlab-ci` as an input and import its +flake module within your `flake-parts` configuration. + +```nix title="flake.nix" +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; # Or your preferred nixpkgs branch/version + flake-parts.url = "github:hercules-ci/flake-parts"; + + # Add nix-gitlab-ci as an input + # recommendation: pin to a specific release/version + nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; + }; + + outputs = { nixpkgs, flake-parts, ... }@inputs: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + # Import the nix-gitlab-ci flake module + inputs.nix-gitlab-ci.flakeModule + ]; + + systems = [ + "x86_64-linux" + "aarch64-linux" + # Add other systems you need + ]; + + perSystem = { pkgs, ... }: { + # define your CI pipelines + # ci = { ... }; + # pipelines."merge_request_event" = { ... }; + }; + }; +} +``` + +Replace `` with the specific version or commit hash of `nix-gitlab-ci` +you wish to use. Pinning to a specific version is highly recommended for +reproducibility and compatibility. + +!!! warning + + While the flake input is locked through `flake.lock`, the CI/CD component + will always use the latest commit of the reference. This means that by using + a branch like `main` as version for both, the CI/CD component will always use + the latest commit while your flake uses a fixed one. + This could result in drift between both, potentially breaking stuff. + +## Including in `.gitlab-ci.yml` + +Your `.gitlab-ci.yml` file will be minimal. Its primary role is to include the +`nix-gitlab-ci` component, which will then generate the full CI configuration +based on your Nix code. + +```yaml title=".gitlab-ci.yml" +include: + - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ + inputs: + # This input sets the Docker image tag used for the CI jobs. + # Use the same version as you pinned in your flake.nix for consistency. + version: +``` + +Again, ensure `` matches the version used in your `flake.nix`. +This component includes a job (`build:nix-ci`) that will evaluate your Nix +configuration and generate the `.gitlab-ci.yml` used for the pipeline run. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..b28a755 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,33 @@ +# Usage + +To create a basic pipeline, configure it by setting `ci` in `perSystem`. +The schema is similar to the `.gitlab-ci.yml`, only jobs are defined differently: + +```nix +ci = { + # Nix GitLab CI specific config, see `configType` in `flakeModule.nix` + config = {}; + jobs = { + "job-a" = {}; + "job-b" = {}; + }; +}; +``` + +For every job, there are a couple of settings you can adjust aswell: + +```nix +"job-a" = { + # see `jobType` in `flakeModule.nix` + nix = { + enable = true; # is this a nix-based job? + deps = []; # dependencies to install for this job + # for gitlab runner cache: + enable-runner-cache = false; + runner-cache-key = ""; + }; +}; +``` + +Since V2 multiple pipelines are supported. +See [Multiple Pipelines](./multi_pipeline.md) for more. diff --git a/docs/utilities.md b/docs/utilities.md new file mode 100644 index 0000000..e5a7b96 --- /dev/null +++ b/docs/utilities.md @@ -0,0 +1,59 @@ +# Utilities + +Nix GitLab CI provides a couple of utilities to help with development and +debugging. + +## Disabling caching temporarily + +Nix GitLab CI often utilizes caching mechanisms to speed up your pipelines +(see [Caching](./caching.md)). +However, there might be situations where you need to temporarily disable these +caches for a specific pipeline run, for example, to debug a caching issue or +ensure a clean build. + +To disable most of the provided caches for a pipeline, set the environment +variable `NIX_CI_DISABLE_CACHE` to any non-empty value (e.g., `yes`, `true`, `1`) +when triggering the pipeline in the GitLab UI or via the API. + +## Forcing a rebuild of the CI pipeline definition + +The job responsible for generating the `.gitlab-ci.yml` from your Nix code +(`build:nix-ci`) might itself be cached. If you've made changes to your Nix CI +configuration and the pipeline doesn't seem to pick them up, the cached job +definition might be the reason. + +You should first double check if all the Nix files you defined the CI config in +are specified in the `cache_files` CI/CD-component input +(see [CI/CD Component](./cicd_component.md) for more). + +To force this specific job to rebuild and re-evaluate your Nix configuration, +set the environment variable `NIX_CI_FORCE_BUILD` when triggering the pipeline. + +## Running jobs locally + +One of the benefits of defining your CI jobs with Nix is the ability to run them +locally in an environment that closely mirrors the CI environment. This can +significantly speed up debugging and development. + +You can run the script of any defined job locally using the `nix run` command. +The syntax is: + +```sh +nix run .#gitlab-ci:pipeline::job: +``` + +Replace `` with the name of the pipeline the job belongs to +(e.g., `default` for jobs defined under the `ci` attribute) and `` +with the name of the job you want to run. + +This command will set up the environment with the specified `nix.deps` and +execute the job's `script`. + +There is also an attribute `.#gitlab-ci:pipeline::job-deps:`. +Building this derivation will generate a shell script which exports the required +environment variables for the job, such as the `PATH` including all dependencies +and any custom environment variables that contain store paths (ensuring they are +correctly resolved across different architectures). + +You can use this to inspect the environment that would be set up for a job without +running the full script. diff --git a/flake.lock b/flake.lock index dd864f4..f1a0875 100644 --- a/flake.lock +++ b/flake.lock @@ -14,11 +14,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1728672398, - "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", + "lastModified": 1737621947, + "narHash": "sha256-8HFvG7fvIFbgtaYAY2628Tb89fA55nPm2jSiNs0/Cws=", "owner": "cachix", "repo": "cachix", - "rev": "aac51f698309fd0f381149214b7eee213c66ef0a", + "rev": "f65a3cd5e339c223471e64c051434616e18cc4f5", "type": "github" }, "original": { @@ -32,18 +32,16 @@ "inputs": { "cachix": "cachix", "flake-compat": "flake-compat", - "git-hooks": [ - "git-hooks" - ], + "git-hooks": "git-hooks", "nix": "nix", "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1732585607, - "narHash": "sha256-6ffeaSMuaL326f7KrCeScpSJtdHsFKS9gPrsSZkndvU=", + "lastModified": 1743687534, + "narHash": "sha256-QIh5jKWE0aZ4N77Zu3kz0RjA22zqqy5p6PfJnOW5rPE=", "owner": "cachix", "repo": "devenv", - "rev": "a520f05c40ebecaf5e17064b27e28ba8e70c49fb", + "rev": "efd5d68a6483573410565d0b940e1a67b6f92591", "type": "github" }, "original": { @@ -55,27 +53,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -111,11 +93,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1730504689, - "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", + "lastModified": 1743550720, + "narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "506278e768c2a08bec68eb62932193e341f55c90", + "rev": "c621e8422220273271f52058f618c94e405bb0f5", "type": "github" }, "original": { @@ -126,17 +108,21 @@ }, "git-hooks": { "inputs": { - "flake-compat": "flake-compat_2", + "flake-compat": [ + "devenv" + ], "gitignore": "gitignore", - "nixpkgs": "nixpkgs_4", - "nixpkgs-stable": "nixpkgs-stable" + "nixpkgs": [ + "devenv", + "nixpkgs" + ] }, "locked": { - "lastModified": 1732021966, - "narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", + "lastModified": 1740849354, + "narHash": "sha256-oy33+t09FraucSZ2rZ6qnD1Y1c8azKKmQuCvF2ytUko=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "3308484d1a443fc5bc92012435d79e80458fe43c", + "rev": "4a709a8ce9f8c08fa7ddb86761fe488ff7858a07", "type": "github" }, "original": { @@ -148,6 +134,7 @@ "gitignore": { "inputs": { "nixpkgs": [ + "devenv", "git-hooks", "nixpkgs" ] @@ -182,6 +169,21 @@ "type": "github" } }, + "mkdocs-material-umami": { + "locked": { + "lastModified": 1745840856, + "narHash": "sha256-1Ad1JTMQMP6YsoIKAA+SBCE15qWrYkGue9/lXOLnu9I=", + "owner": "technofab", + "repo": "mkdocs-material-umami", + "rev": "3ac9b194450f6b779c37b8d16fec640198e5cd0a", + "type": "gitlab" + }, + "original": { + "owner": "technofab", + "repo": "mkdocs-material-umami", + "type": "gitlab" + } + }, "nix": { "inputs": { "flake-compat": [ @@ -201,11 +203,11 @@ ] }, "locked": { - "lastModified": 1727438425, - "narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=", + "lastModified": 1741798497, + "narHash": "sha256-E3j+3MoY8Y96mG1dUIiLFm2tZmNbRvSiyN7CrSKuAVg=", "owner": "domenkozar", "repo": "nix", - "rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546", + "rev": "f3f44b2baaf6c4c6e179de8cbb1cc6db031083cd", "type": "github" }, "original": { @@ -215,13 +217,30 @@ "type": "github" } }, + "nix-mkdocs": { + "locked": { + "dir": "lib", + "lastModified": 1745175318, + "narHash": "sha256-eJOTw3SK4psqBPP2hNJ4D/13/oi/FLoM0tYKoCGVFP8=", + "owner": "technofab", + "repo": "nixmkdocs", + "rev": "6d6e0139060c896ae14de4b9c82335655a384643", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "technofab", + "repo": "nixmkdocs", + "type": "gitlab" + } + }, "nixpkgs": { "locked": { - "lastModified": 1730531603, - "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", + "lastModified": 1733212471, + "narHash": "sha256-M1+uCoV5igihRfcUKrr1riygbe73/dzNnzPsmaLCmpo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", + "rev": "55d15ad12a74eb7d4646254e13638ad0c4128776", "type": "github" }, "original": { @@ -233,29 +252,16 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1730504152, - "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" - }, - "original": { - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" - } - }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1730741070, - "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", + "lastModified": 1743296961, + "narHash": "sha256-b1EdN3cULCqtorQ4QeWgLMrd5ZGOjLSLemfa00heasc=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "e4822aea2a6d1cdd36653c134cacfd64c97ff4fa", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", + "owner": "nix-community", + "repo": "nixpkgs.lib", "type": "github" } }, @@ -277,11 +283,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1716977621, - "narHash": "sha256-Q1UQzYcMJH4RscmpTkjlgqQDX5yi1tZL0O345Ri6vXQ=", + "lastModified": 1733477122, + "narHash": "sha256-qamMCz5mNpQmgBwc8SB5tVMlD5sbwVIToVZtSxMph9s=", "owner": "cachix", "repo": "devenv-nixpkgs", - "rev": "4267e705586473d3e5c8d50299e71503f16a6fb6", + "rev": "7bd9e84d0452f6d2e63b6e6da29fe73fac951857", "type": "github" }, "original": { @@ -293,11 +299,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1730768919, - "narHash": "sha256-8AKquNnnSaJRXZxc5YmF/WfmxiHX6MMZZasRP6RRQkE=", + "lastModified": 1745377448, + "narHash": "sha256-jhZDfXVKdD7TSEGgzFJQvEEZ2K65UMiqW5YJ2aIqxMA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a04d33c0c3f1a59a2c1cb0c6e34cd24500e5a1dc", + "rev": "507b63021ada5fee621b6ca371c4fca9ca46f52c", "type": "github" }, "original": { @@ -309,27 +315,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1732238832, - "narHash": "sha256-sQxuJm8rHY20xq6Ah+GwIUkF95tWjGRd1X8xF+Pkk38=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "8edf06bea5bcbee082df1b7369ff973b91618b8d", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_6": { - "locked": { - "lastModified": 1731890469, - "narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=", + "lastModified": 1735554305, + "narHash": "sha256-zExSA1i/b+1NMRhGGLtNfFGXgLtgo+dcuzHzaWA6w3Q=", "owner": "nixos", "repo": "nixpkgs", - "rev": "5083ec887760adfe12af64830a66807423a859a7", + "rev": "0e82ab234249d8eee3e8c91437802b32c74bb3fd", "type": "github" }, "original": { @@ -343,8 +333,9 @@ "inputs": { "devenv": "devenv", "flake-parts": "flake-parts_2", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_5", + "mkdocs-material-umami": "mkdocs-material-umami", + "nix-mkdocs": "nix-mkdocs", + "nixpkgs": "nixpkgs_4", "systems": "systems", "treefmt-nix": "treefmt-nix" } @@ -366,14 +357,14 @@ }, "treefmt-nix": { "inputs": { - "nixpkgs": "nixpkgs_6" + "nixpkgs": "nixpkgs_5" }, "locked": { - "lastModified": 1732643199, - "narHash": "sha256-uI7TXEb231o8dkwB5AUCecx3AQtosRmL6hKgnckvjps=", + "lastModified": 1743677901, + "narHash": "sha256-eWZln+k+L/VHO69tUTzEmgeDWNQNKIpSUa9nqQgBrSE=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "84637a7ab04179bdc42aa8fd0af1909fba76ad0c", + "rev": "57dabe2a6255bd6165b2437ff6c2d1f6ee78421a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 596b693..6753d41 100644 --- a/flake.nix +++ b/flake.nix @@ -8,14 +8,15 @@ imports = [ inputs.devenv.flakeModule inputs.treefmt-nix.flakeModule + inputs.nix-mkdocs.flakeModule ./lib/flakeModule.nix ]; systems = import systems; flake = {}; perSystem = { pkgs, - inputs', config, + system, ... }: rec { treefmt = { @@ -25,6 +26,15 @@ mdformat.enable = true; yamlfmt.enable = true; }; + settings.formatter = { + yamlfmt.excludes = ["templates/nix-gitlab-ci.yml"]; + mdformat.command = let + pkg = pkgs.python3.withPackages (p: [ + p.mdformat + p.mdformat-mkdocs + ]); + in "${pkg}/bin/mdformat"; + }; }; devenv.shells.default = { containers = pkgs.lib.mkForce {}; @@ -39,16 +49,103 @@ }; }; }; + doc = { + path = ./docs; + deps = pp: [ + pp.mkdocs-material + (pp.callPackage inputs.mkdocs-material-umami {}) + ]; + config = { + site_name = "Nix GitLab CI"; + repo_name = "TECHNOFAB/nix-gitlab-ci"; + repo_url = "https://gitlab.com/TECHNOFAB/nix-gitlab-ci"; + edit_uri = "edit/main/docs/"; + theme = { + name = "material"; + features = ["content.code.copy" "content.action.edit"]; + icon.repo = "simple/gitlab"; + logo = "images/logo.png"; + favicon = "images/favicon.png"; + palette = [ + { + scheme = "default"; + media = "(prefers-color-scheme: light)"; + primary = "deep orange"; + accent = "orange"; + toggle = { + icon = "material/brightness-7"; + name = "Switch to dark mode"; + }; + } + { + scheme = "slate"; + media = "(prefers-color-scheme: dark)"; + primary = "deep orange"; + accent = "orange"; + toggle = { + icon = "material/brightness-4"; + name = "Switch to light mode"; + }; + } + ]; + }; + plugins = ["search" "material-umami"]; + nav = [ + {"Introduction" = "index.md";} + {"Setup" = "setup.md";} + {"Usage" = "usage.md";} + {"CI/CD Component" = "cicd_component.md";} + {"Environment Variables" = "environment_variables.md";} + {"Caching" = "caching.md";} + {"Multiple Pipelines" = "multi_pipeline.md";} + {"Utilities" = "utilities.md";} + {"Kubernetes Runner Example" = "kubernetes_runner.md";} + {"Example Configs" = "examples.md";} + ]; + markdown_extensions = [ + { + "pymdownx.highlight".pygments_lang_class = true; + } + "pymdownx.inlinehilite" + "pymdownx.snippets" + "pymdownx.superfences" + "fenced_code" + "admonition" + ]; + extra.analytics = { + provider = "umami"; + site_id = "28f7c904-db22-4c2b-9ee4-ed42e14b6db9"; + src = "https://analytics.tf/umami"; + domains = "nix-gitlab-ci.projects.tf"; + feedback = { + title = "Was this page helpful?"; + ratings = [ + { + icon = "material/thumb-up-outline"; + name = "This page is helpful"; + data = "good"; + note = "Thanks for your feedback!"; + } + { + icon = "material/thumb-down-outline"; + name = "This page could be improved"; + data = "bad"; + note = "Thanks for your feedback! Please leave feedback by creating an issue :)"; + } + ]; + }; + }; + }; + }; + # should set the "default" pipeline ci = { - # use the image built in the parent pipeline for dogfooding - config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA"; - stages = ["test"]; + stages = ["test" "build" "deploy"]; jobs = { "test" = { stage = "test"; nix = { deps = [pkgs.hello pkgs.curl]; - disable-cache = false; + enable-runner-cache = true; }; variables = { TEST = "test"; @@ -60,6 +157,11 @@ "echo $TEST $TEST_WITH_DERIVATION" ]; }; + "test-default" = { + stage = "test"; + nix.deps = [pkgs.hello]; + script = ["hello"]; + }; "test-non-nix" = { nix.enable = false; stage = "test"; @@ -68,132 +170,89 @@ "echo \"This job will not be modified to use nix\"" ]; }; + # -- actually useful jobs -- + "docs" = { + stage = "build"; + script = [ + # sh + '' + nix build .#docs:default + mkdir -p public + cp -r result/. public/ + '' + ]; + artifacts.paths = ["public"]; + }; + "pages" = { + nix.enable = false; + image = "alpine:latest"; + stage = "deploy"; + script = ["true"]; + artifacts.paths = ["public"]; + rules = [ + { + "if" = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"; + } + ]; + }; + }; + }; + pipelines."non-default" = { + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + script = [ + "echo Hello from another pipeline" + ]; + }; }; }; packages = let - setupScript = extra_setup: - pkgs.writeShellScriptBin "setup_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" - nix path-info --all > /tmp/nix-store-before - - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${extra_setup} - else - echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration" - fi - - export NIX_CONFIG=" - extra-trusted-public-keys = $NIX_PUBLIC_KEYS - extra-trusted-substituters = $NIX_SUBSTITUTERS - extra-substituters = $NIX_SUBSTITUTERS - $NIX_CONFIG - $NIX_EXTRA_CONFIG - " - echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" - - # load the job's deps only if the name was passed - if [[ ! -z $1 ]]; then - echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job" - nix build .#gitlab-ci-job-deps:$1 - source $(readlink -f result) - echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" - fi - ''; - finalizeScript = push_command: - pkgs.writeShellScriptBin "finalize_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache" - nix path-info --all > /tmp/nix-store-after - ${pkgs.diffutils}/bin/diff --new-line-format="%L" \ - --old-line-format="" --unchanged-line-format="" \ - /tmp/nix-store-before /tmp/nix-store-after \ - | { - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${push_command} - else - ${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; } - fi - } - echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" - ''; - mkImage = extraPackages: - pkgs.dockerTools.buildImage { - name = "nix-gitlab-ci"; - fromImage = pkgs.dockerTools.pullImage { - imageName = "nixpkgs/nix-flakes"; - imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; - sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; - finalImageName = "nixpkgs/nix-flakes"; - finalImageTag = "latest"; - }; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = - [ - pkgs.gitMinimal - pkgs.gnugrep - ] - ++ extraPackages; - pathsToLink = ["/bin"]; - }; - }; + setupScript = pkgs.writeShellScriptBin "setup_nix_ci" (builtins.readFile ./scripts/setup_nix_ci.sh); + finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" (builtins.readFile ./scripts/finalize_nix_ci.sh); in { - setup-script = - setupScript - # sh - '' - # extra_setup - true - ''; - finalize-script = - finalizeScript - # sh - '' - # push_command - true - ''; - image = mkImage [ - (setupScript - # sh - '' - cachedir="$(pwd)/.nix-cache" - echo "Configuring caching with the Runner Cache in $cachedir..." - export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true" - '') - (finalizeScript - # sh - '' - # add ^* to all store paths ending in .drv (prevent warning log spam) - ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true - '') - ]; - image-cachix = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with cachix..." - ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true - '') - (finalizeScript - # sh - '' - ${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true - '') - ]; - image-attic = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with attic..." - ${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true - ${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true - '') - (finalizeScript - # sh - '' - ${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true - '') - ]; + setup-script = setupScript; + finalize-script = finalizeScript; + image = pkgs.dockerTools.buildImage { + name = "nix-ci"; + fromImage = let + hashes = { + "x86_64-linux" = "sha256-kJ7dqje5o1KPr3RDZ7/THbhMSoiCU1C/7HshDrNfwnM="; + "aarch64-linux" = "sha256-jz+Z3Ji+hy5d9ImOh/YOKCqy9P9/cseSov+5J/O95bg="; + }; + # check digest of tags like nixos-24.11-aarch64-linux etc. + digests = { + "x86_64-linux" = "sha256:345f210dea4cbd049e2d01d13159c829066dfb6e273cdd49ea878186d17b19f7"; + "aarch64-linux" = "sha256:66163fdf446d851416dd4e9be28c0794d9c2550214a57a846957699a3f5747f6"; + }; + hash = hashes.${system} or (throw "Unsupported system"); + imageDigest = digests.${system} or (throw "Unsupported system"); + in + pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-flakes"; + inherit hash imageDigest; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = with pkgs; + [ + gitMinimal + gnugrep + gnused + coreutils + diffutils + cachix + attic-client + ] + ++ [ + setupScript + finalizeScript + ]; + pathsToLink = ["/bin"]; + }; + }; }; checks = packages; @@ -206,12 +265,10 @@ # flake & devenv related flake-parts.url = "github:hercules-ci/flake-parts"; systems.url = "github:nix-systems/default-linux"; - devenv = { - url = "github:cachix/devenv"; - inputs.git-hooks.follows = "git-hooks"; - }; - git-hooks.url = "github:cachix/git-hooks.nix"; + devenv.url = "github:cachix/devenv"; treefmt-nix.url = "github:numtide/treefmt-nix"; + nix-mkdocs.url = "gitlab:technofab/nixmkdocs?dir=lib"; + mkdocs-material-umami.url = "gitlab:technofab/mkdocs-material-umami"; }; nixConfig = { diff --git a/gitlab-ci.yml b/gitlab-ci.yml deleted file mode 100644 index f1cb469..0000000 --- a/gitlab-ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# NOTE: DEPRECATED: please switch to the CI/CD Component or include "templates/nix-gitlab-ci.yml" -# -variables: - # latest | latest-cachix | latest-attic etc. - NIX_CI_IMAGE_TAG: latest -stages: - - build - - trigger -nix-ci:build: - stage: build - image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${NIX_CI_IMAGE_TAG} - before_script: - - source setup_nix_ci - script: - # build the generated-gitlab-ci.yml - - nix build .#gitlab-ci-config - - install result generated-gitlab-ci.yml - after_script: - # upload to binary cache - - finalize_nix_ci - artifacts: - paths: - - generated-gitlab-ci.yml -nix-ci:trigger: - stage: trigger - needs: - - nix-ci:build - trigger: - include: - - artifact: generated-gitlab-ci.yml - job: nix-ci:build - strategy: depend diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 64f7926..469fc3e 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,137 +9,159 @@ pkgs, ... }: let + inherit (lib) isAttrs filterAttrs mapAttrs types mkOption toList; cfg = config.ci.config; + stdenvMinimal = pkgs.stdenvNoCC.override { + cc = null; + preHook = ""; + allowedRequisites = null; + initialPath = [pkgs.coreutils pkgs.findutils]; + shell = "/bin/sh"; + extraNativeBuildInputs = []; + }; + filterAttrsRec = pred: v: - if lib.isAttrs v - then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v) + if isAttrs v + then filterAttrs pred (mapAttrs (path: filterAttrsRec pred) v) else v; - subType = options: lib.types.submodule {inherit options;}; + subType = options: types.submodule {inherit options;}; mkNullOption = type: - lib.mkOption { + mkOption { default = null; - type = lib.types.nullOr type; + type = types.nullOr type; }; - configType = with lib; - subType { - default-nix-image = mkOption { - type = types.str; - default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest"; - description = "The image to use on nix jobs"; - }; - nix-jobs-per-default = mkOption { - type = types.bool; - default = true; - description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; - }; - disable-cache = mkOption { - type = types.bool; - default = false; - description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; - description = "Cache key to use for the nix cache"; - }; + configType = subType { + nix-jobs-by-default = mkOption { + type = types.bool; + default = true; + description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; }; - jobType = with lib; - subType { - # nix ci opts - nix = mkOption { - type = subType { - enable = mkOption { - type = types.bool; - default = cfg.nix-jobs-per-default; - description = "Handle this job as a nix job"; - }; - deps = mkOption { - type = types.listOf types.package; - default = []; - description = "Dependencies/packages to install for this job"; - }; - disable-cache = mkOption { - type = types.bool; - default = cfg.disable-cache; - description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = cfg.cache-key; - description = "Cache key to use for the nix cache"; - }; - }; - default = {}; - description = "Configure Nix Gitlab CI for each job individually"; - }; - # gitlab opts - script = mkOption { - type = types.listOf types.str; - default = []; - }; - stage = mkOption { - type = types.str; - default = "test"; - }; - image = mkOption { - type = types.str; - default = cfg.default-nix-image; - }; - after_script = mkNullOption (types.listOf types.str); - allow_failure = mkNullOption (types.either types.attrs types.bool); - artifacts = mkNullOption (types.attrs); - before_script = mkNullOption (types.listOf types.str); - cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); - coverage = mkNullOption (types.str); - dependencies = mkNullOption (types.listOf types.str); - environment = mkNullOption (types.either types.attrs types.str); - extends = mkNullOption (types.str); - hooks = mkNullOption (types.attrs); - id_tokens = mkNullOption (types.attrs); - "inherit" = mkNullOption (types.attrs); - interruptible = mkNullOption (types.bool); - needs = mkNullOption (types.listOf (types.either types.str types.attrs)); - publish = mkNullOption (types.str); - pages = mkNullOption (types.attrs); - parallel = mkNullOption (types.either types.int types.attrs); - release = mkNullOption (types.attrs); - retry = mkNullOption (types.either types.int types.attrs); - rules = mkNullOption (types.listOf types.attrs); - resource_group = mkNullOption (types.str); - secrets = mkNullOption (types.attrs); - services = mkNullOption (types.listOf types.attrs); - start_in = mkNullOption (types.str); - tags = mkNullOption (types.listOf types.str); - timeout = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - when = mkNullOption (types.str); - }; - in { - options = with lib; { - ci = mkOption { + }; + jobType = subType { + # nix ci opts + nix = mkOption { type = subType { - config = mkOption { - type = configType; - description = '' - Configuration options for the nix part itself - ''; - default = {}; + enable = mkOption { + type = types.bool; + default = cfg.nix-jobs-by-default; + description = "Handle this job as a nix job"; }; - image = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - default = mkNullOption (types.attrs); - stages = mkNullOption (types.listOf types.str); - include = mkNullOption (types.attrs); - workflow = mkNullOption (types.attrs); - jobs = mkOption { - type = types.lazyAttrsOf jobType; - default = {}; + deps = mkOption { + type = types.listOf types.package; + default = []; + description = "Dependencies/packages to install for this job"; + }; + enable-runner-cache = mkOption { + type = types.bool; + default = false; + description = '' + Cache this job using the GitLab Runner cache. + Warning: useful for tiny jobs, but most of the time it just takes an eternity. + ''; + }; + runner-cache-key = mkOption { + type = types.str; + default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; + description = "Cache key to use for the runner nix cache. Requires enable-runner-cache = true"; }; }; + default = {}; + description = "Configure Nix Gitlab CI for each job individually"; + }; + # gitlab opts + script = mkOption { + type = types.listOf types.str; + default = []; + }; + stage = mkOption { + type = types.str; + default = "test"; + }; + image = mkOption { + type = types.str; + default = "$NIX_CI_IMAGE"; + }; + after_script = mkNullOption (types.listOf types.str); + allow_failure = mkNullOption (types.either types.attrs types.bool); + artifacts = mkNullOption (types.attrs); + before_script = mkNullOption (types.listOf types.str); + cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); + coverage = mkNullOption (types.str); + dependencies = mkNullOption (types.listOf types.str); + environment = mkNullOption (types.either types.attrs types.str); + extends = mkNullOption (types.str); + hooks = mkNullOption (types.attrs); + id_tokens = mkNullOption (types.attrs); + "inherit" = mkNullOption (types.attrs); + interruptible = mkNullOption (types.bool); + needs = mkNullOption (types.listOf (types.either types.str types.attrs)); + publish = mkNullOption (types.str); + pages = mkNullOption (types.attrs); + parallel = mkNullOption (types.either types.int types.attrs); + release = mkNullOption (types.attrs); + retry = mkNullOption (types.either types.int types.attrs); + rules = mkNullOption (types.listOf types.attrs); + resource_group = mkNullOption (types.str); + secrets = mkNullOption (types.attrs); + services = mkNullOption (types.listOf types.attrs); + start_in = mkNullOption (types.str); + tags = mkNullOption (types.listOf types.str); + timeout = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + when = mkNullOption (types.str); + }; + + ciType = subType { + config = mkOption { + type = configType; description = '' + Configuration options for the nix part itself + ''; + default = {}; + }; + image = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + default = mkNullOption (types.attrs); + stages = mkNullOption (types.listOf types.str); + include = mkNullOption (types.attrs); + workflow = mkNullOption (types.attrs); + jobs = mkOption { + type = types.lazyAttrsOf jobType; + default = {}; + }; + }; + in { + options = { + pipelines = mkOption { + type = types.lazyAttrsOf ciType; + description = '' + Create multiple GitLab CI pipelines. + + See README.md for more information about how a pipeline is selected. + ''; + default = {}; + apply = op: let + # NOTE: show warning if "default" is set and config.ci is not {} + legacyMode = config.ci != {}; + defaultExists = builtins.hasAttr "default" op; + value = + { + "default" = config.ci; + } + // op; + in + if defaultExists && legacyMode + then builtins.trace "Warning: config.ci is overwritten by pipelines.default" value + else value; + }; + ci = mkOption { + type = ciType; + description = '' + Note: this is a shorthand for writing `pipelines."default"` + Generate a Gitlab CI configuration which can be used to trigger a child pipeline. This will inject code which pre-downloads the nix deps before each job and adds them to PATH. ''; @@ -148,7 +170,10 @@ }; config.legacyPackages = let - toYaml = (pkgs.formats.yaml {}).generate; + # NOTE: json is also valid yaml and this removes dependency on jq + # and/or remarshal (used in pkgs.formats.json and pkgs.formats.yaml + # respectively) + toYaml = name: value: builtins.toFile name (builtins.toJSON value); mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); prepend = key: arr: job: job @@ -175,94 +200,111 @@ } ) (job.variables or {}); - - jobs = filterAttrsRec (n: v: v != null) config.ci.jobs; - rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job-deps:${key}"; - value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' - export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; - ${variableExports} - ''; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables false job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job:${key}"; - value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - # set up deps and environment variables containing store paths - . ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - ${lib.concatLines (job.before_script or [])} - echo -e "\e[32mRunning script...\e[0m" - ${lib.concatLines job.script} - echo -e "\e[32mRunning after_script...\e[0m" - ${lib.concatLines (job.after_script or [])} - ''; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - mapAttrs (key: job: { - name = key; - value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - builtins.removeAttrs ( - (prependToBeforeScript [ - "source setup_nix_ci ${key}" - ] - (appendToAfterScript [ - "finalize_nix_ci" - ] - job)) - // lib.optionalAttrs job.nix.enable { - image = job.image; - variables = - (filterJobVariables false job) - // lib.optionalAttrs job.nix.disable-cache { - NIX_CI_DISABLE_CACHE = "yes"; - }; - cache = - ( - let - c = job.cache or []; - in - if builtins.isList c - then c - else [c] - ) - ++ (lib.optional (!job.nix.disable-cache) { - key = job.nix.cache-key; - paths = [".nix-cache/"]; - }); - } - ) ["nix"]; - }) - jobs; in - { - gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript; + lib.fold (pipeline: acc: acc // pipeline) {} (map ( + pipeline_name: let + pipeline = config.pipelines."${pipeline_name}"; + jobs = filterAttrsRec (n: v: v != null) pipeline.jobs; + rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs pipeline ["jobs" "config"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + mapAttrs (key: job: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"; + value = stdenvMinimal.mkDerivation { + name = "gitlab-ci-job-deps-${key}"; + dontUnpack = true; + installPhase = let + script = '' + export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; + # variables containing nix derivations: + ${variableExports} + ''; + in + # sh + '' + echo '${script}' > $out + chmod +x $out + ''; + }; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + mapAttrs (key: job: let + variablesWithoutStorePaths = filterJobVariables false job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths + ); + in { + name = "gitlab-ci:pipeline:${pipeline_name}:job:${key}"; + value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + # set up deps and environment variables containing store paths + . ${jobsMappedForDeps."gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}"} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + ${lib.concatLines (job.before_script or [])} + echo -e "\e[32mRunning script...\e[0m" + ${lib.concatLines job.script} + echo -e "\e[32mRunning after_script...\e[0m" + ${lib.concatLines (job.after_script or [])} + ''; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + mapAttrs (key: job: { + name = key; + value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + builtins.removeAttrs ( + (prependToBeforeScript [ + "source setup_nix_ci \"gitlab-ci:pipeline:${pipeline_name}:job-deps:${key}\"" + ] + (appendToAfterScript [ + "finalize_nix_ci" + ] + job)) + // lib.optionalAttrs job.nix.enable { + image = job.image; + variables = + (filterJobVariables false job) + // lib.optionalAttrs job.nix.enable-runner-cache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + cache = + ( + let + c = job.cache or []; + in + toList c + ) + ++ (lib.optional (job.nix.enable-runner-cache) { + key = job.nix.runner-cache-key; + paths = [".nix-cache/"]; + }); + } + ) ["nix"]; + }) + jobs; + in + # gitlab-ci:pipeline: + # gitlab-ci:pipeline::job: + # gitlab-ci:pipeline::job-deps: + { + "gitlab-ci:pipeline:${pipeline_name}" = toYaml "gitlab-ci-${pipeline_name}.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript + ) (builtins.attrNames config.pipelines)); } ); } diff --git a/scripts/finalize_nix_ci.sh b/scripts/finalize_nix_ci.sh new file mode 100644 index 0000000..baa6a0d --- /dev/null +++ b/scripts/finalize_nix_ci.sh @@ -0,0 +1,41 @@ +echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci[collapsed=true]\\r\\e[0KFinalizing Nix CI..." + nix path-info --all > /tmp/nix-store-after + echo "Finding new paths..." + NEW_PATHS=$(diff --new-line-format="%L" \ + --old-line-format="" --unchanged-line-format="" \ + /tmp/nix-store-before /tmp/nix-store-after) + COUNT=$(wc -l <<<"$NEW_PATHS") + + if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then + export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}"; + fi + + if [ -z "$NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($NIX_CI_CACHE_STRATEGY)" + echo -n "$NEW_PATHS" | { + case "$NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + # add ^* to all store paths ending in .drv (prevent warning log spam) + sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true + ;; + "attic") + attic push --stdin ci:$ATTIC_CACHE || true + ;; + "cachix") + cachix push $CACHIX_CACHE || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'" + ;; + esac + } + echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" + else + echo "Caching disabled, not uploading $COUNT new store entries..." + fi +echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K" + diff --git a/scripts/setup_nix_ci.sh b/scripts/setup_nix_ci.sh new file mode 100644 index 0000000..1f2af66 --- /dev/null +++ b/scripts/setup_nix_ci.sh @@ -0,0 +1,49 @@ +echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" + nix path-info --all > /tmp/nix-store-before + + if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then + export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}"; + echo "NIX_CI_CACHE_STRATEGY was set to auto, selected '$NIX_CI_CACHE_STRATEGY' for this job" + fi + + if [ -z "$NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($NIX_CI_CACHE_STRATEGY)" + case "$NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + echo "Runner Cache: $RUNNER_CACHE" + export NIX_CONFIG="$NIX_CONFIG + extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true + extra-substituters = $RUNNER_CACHE?priority=10&trusted=true + " + ;; + "attic") + echo "Attic Cache: $ATTIC_CACHE" + attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true + attic use "$ATTIC_CACHE" || true + ;; + "cachix") + echo "Cachix Cache: $CACHIX_CACHE" + cachix use "$CACHIX_CACHE" || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'" + ;; + esac + echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K" + else + echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..." + fi + + # load the job's deps only if the name was passed + if [[ ! -z $1 ]]; then + echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job" + nix build .#$1 + source $(readlink -f result) + echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" + fi +echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + diff --git a/templates/nix-gitlab-ci.yml b/templates/nix-gitlab-ci.yml index 911556a..cc87788 100644 --- a/templates/nix-gitlab-ci.yml +++ b/templates/nix-gitlab-ci.yml @@ -1,63 +1,103 @@ spec: inputs: - image_tag: + cache_strategy: type: string - description: "latest | latest-cachix | latest-attic etc." - default: latest + description: | + auto (default) | none | runner | cachix | attic + Sets the default caching strategy. + - "auto": dynamically selects the best strategy for every job based on env variables + - "none": disables caching + - "runner", "cachix" & "attic": forces every job to use this strategy + + Can be overridden by setting NIX_CI_CACHE_STRATEGY in the pipeline variables. + default: "auto" cache_files: type: array description: | Files to use as the cache key for the generated pipeline yaml. - If you use "ci.nix" to define CI, add that here for example + If you use "ci.nix" to define CI, add that here for example. + Note that max 2 items are allowed in cache:key:files, so use something like + ["flake.*", "ci.nix"] f. ex. to match flake.lock, flake.nix and ci.nix. default: ["flake.nix", "flake.lock"] - disable_cache: + version: type: string description: | - Disables any caching provided by this component. Set to any non-empty value to disable caching. - default: "" + Which version of the Nix CI image to use. Using a tag/version is recommended. --- stages: - build - trigger variables: - NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]${NIX_CI_DISABLE_CACHE:-}" + # These can be overriden, see https://docs.gitlab.com/ci/variables/#cicd-variable-precedence + # which image should be used by default. + NIX_CI_IMAGE: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.version ]] + # default cache stategy + NIX_CI_CACHE_STRATEGY: $[[ inputs.cache_strategy ]] nix-ci:build: stage: build - image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]] + image: $NIX_CI_IMAGE cache: - key: files: $[[ inputs.cache_files ]] paths: - - generated-gitlab-ci.yml + - .nix-ci-pipelines/ - key: nix paths: - .nix-cache/ before_script: - # generated-gitlab-ci.yml exists in the cache - - '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true' - # allow the user to manually skip the cache (when the key files are not correctly configured etc.) - - '[ -n "$NIX_CI_FORCE_BUILD" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true' - # only setup when we need to generate the pipeline yaml - - 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi' + - | + # if no explicit pipeline is requested + if [[ -z "${NIX_CI_PIPELINE_NAME:-}" ]]; then + # if regex matches, use pipeline "default", otherwise $CI_PIPELINE_SOURCE + [[ "${CI_PIPELINE_SOURCE}" =~ ${NIX_CI_DEFAULT_SOURCES:-.*} ]] \ + && NIX_CI_PIPELINE_NAME="default" \ + || NIX_CI_PIPELINE_NAME="$CI_PIPELINE_SOURCE"; + fi + echo "NIX_CI_GENERATED_PIPELINE_NAME=$NIX_CI_PIPELINE_NAME" >> trigger.env + # inheritance of pipeline variables is a bit weird, so explicitly override them + # (ctx: setting any of these in the project variables would only apply correctly + # in this pipeline, not the child pipeline, instead weirdly enough the default + # variables above are used). If any other variables are added at the top, add them + # here aswell + echo "NIX_CI_IMAGE=$NIX_CI_IMAGE" >> trigger.env + echo "NIX_CI_CACHE_STRATEGY=$NIX_CI_CACHE_STRATEGY" >> trigger.env + + mkdir -p .nix-ci-pipelines/ + # generated-gitlab-ci.yml exists in the cache + [[ -f ".nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml" ]] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true + # allow the user to manually skip the cache (when the key files are not correctly configured etc.) + [[ -n "$NIX_CI_FORCE_BUILD" ]] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true + + # only setup when we need to generate the pipeline yaml + if [[ -z "$CACHED" ]]; then + source setup_nix_ci; + fi script: - # build the generated-gitlab-ci.yml if it does not exist in the cache - - 'if [ -z "$CACHED" ]; then nix build .#gitlab-ci-config && install result generated-gitlab-ci.yml; fi' + # build the pipeline if it does not exist in the cache + - > + if [[ -z "$CACHED" ]]; then + nix build .#gitlab-ci:pipeline:${NIX_CI_PIPELINE_NAME} && install result .nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml; + fi after_script: - # NOTE: environment variables of before_script and script don't exist here anymore - # # save to binary cache or Gitlab CI cache only if we actually built something # check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called - - 'if [ -f "/tmp/nix-store-before" ]; then finalize_nix_ci; fi' + - | + if [[ -f "/tmp/nix-store-before" ]]; then + finalize_nix_ci; + fi artifacts: paths: - - generated-gitlab-ci.yml + - .nix-ci-pipelines/ + reports: + dotenv: trigger.env + nix-ci:trigger: stage: trigger needs: - nix-ci:build trigger: include: - - artifact: generated-gitlab-ci.yml + - artifact: .nix-ci-pipelines/${NIX_CI_GENERATED_PIPELINE_NAME}.yml job: nix-ci:build strategy: depend forward: