diff --git a/.envrc b/.envrc index a2d9328..565a52a 100644 --- a/.envrc +++ b/.envrc @@ -1,4 +1,2 @@ -if ! use flake . --impure -then - echo "devenv could not be build. The devenv environment was not loaded. Make the necessary changes to flake.nix and hit enter to try again." >&2 -fi +source $(fetchurl https://gitlab.com/rensa-nix/direnv/-/raw/v0.3.0/direnvrc "sha256-u7+KEz684NnIZ+Vh5x5qLrt8rKdnUNexewBoeTcEVHQ=") +use ren //repo/devShells/default diff --git a/.gitignore b/.gitignore index 2a8dde4..fe07db9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,2 @@ -.idea -.devenv -.direnv -.pre-commit-config.yaml +.ren/ result diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0393fb..e8c146c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,35 +1,59 @@ -include: - - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHA - inputs: - image_tag: $CI_COMMIT_SHORT_SHA -stages: - - build-images - - build - - trigger +# Generated by soonix, DO NOT EDIT build:image: - stage: build-images + after_script: + - install -D result dist/nix-ci-$ARCH.tar.gz + artifacts: + paths: + - dist + image: nixpkgs/nix-flakes:latest parallel: matrix: - - VARIANT: ["", "-cachix", "-attic"] - image: nixpkgs/nix-flakes:latest - before_script: - - nix profile install nixpkgs#skopeo - - export PATH="$PATH:$HOME/.nix-profile/bin" + - ARCH: + - x86_64-linux + - aarch64-linux script: - - nix build .#image${VARIANT} - - export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-} - - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_SHORT_SHA}${VARIANT}" - # branches - - | - if [ -z "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${NORMALIZED_BRANCH/main/latest}${VARIANT}"; - fi - # tags - - | - if [ -n "$CI_COMMIT_TAG" ]; then - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ - "docker-archive:result" \ - "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_TAG}${VARIANT}"; - fi + - nix build .#image --system $ARCH + stage: build-images +deploy:image: + before_script: + - 'nix profile install nixpkgs#buildah + + export PATH="$PATH:$HOME/.nix-profile/bin" + + export REGISTRY_AUTH_FILE=${HOME}/auth.json + + echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin + $CI_REGISTRY + + mkdir -p /etc/containers && echo ''{"default":[{"type":"insecureAcceptAnything"}]}'' + > /etc/containers/policy.json + + mkdir -p /var/tmp + + ' + image: nixpkgs/nix-flakes:latest + needs: + - build:image + script: + - "export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\\//-}\nbuildah manifest create localhost/nix-ci\n\ + buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz\n\ + buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz\n\ + buildah manifest push --all localhost/nix-ci docker://${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_SHORT_SHA}\n\ + # branches\nif [ -z \"$CI_COMMIT_TAG\" ]; then\n buildah manifest push --all\ + \ localhost/nix-ci docker://${CI_REGISTRY_IMAGE}/nix-ci:${NORMALIZED_BRANCH/main/latest}\n\ + fi\n# tags\nif [ -n \"$CI_COMMIT_TAG\" ]; then\n buildah manifest push --all\ + \ localhost/nix-ci docker://${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_TAG}\nfi\n" + stage: build-images +include: +- component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHORT_SHA + inputs: + cache_files: + - flake.* + - nix/repo/ci.nix + version: $CI_COMMIT_SHORT_SHA +stages: +- build-images +- build +- trigger +variables: + NIX_CI_IMAGE: $CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA diff --git a/README.md b/README.md index b9a8159..c5e6b92 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -# Nix Gitlab CI +# Nix GitLab CI + +[![built with nix](https://img.shields.io/static/v1?logo=nixos&logoColor=white&label=&message=Built%20with%20Nix&color=41439a)](https://builtwithnix.org) +[![pipeline status](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/badges/main/pipeline.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/commits/main) +![License: MIT](https://img.shields.io/gitlab/license/technofab/nix-gitlab-ci) +[![Latest Release](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/badges/release.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/releases) +[![Support me](https://img.shields.io/badge/Support-me-orange)](https://tec.tf/#support) +[![Docs](https://img.shields.io/badge/Read-Docs-orange)](https://nix-gitlab-ci.projects.tf) Flake module which allows generating a `.gitlab-ci.yml` from Nix. @@ -6,13 +13,13 @@ This allows easily using any Nix package in CI. Also makes it possible to split CI parts in a separate module which can be imported in multiple projects. -## Usage +## Usage (with flake-parts) ```nix # flake.nix { ... - inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; + inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; # recommendation: pin to the latest release/version outputs = {...}: flake-parts.lib.mkFlake {...} { imports = [ @@ -22,16 +29,26 @@ Also makes it possible to split CI parts in a separate module which can be impor perSystem = {pkgs, ...}: { ci = { - stages = ["test"]; - jobs = { - "test" = { - stage = "test"; - nix.deps = [pkgs.unixtools.ping]; - script = [ - "ping -c 5 8.8.8.8" - ]; + config = { + # configure Nix-GitLab-CI here, see docs for options + }; + pipelines."default" = { + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + nix.deps = [pkgs.unixtools.ping]; + script = [ + "ping -c 5 8.8.8.8" + ]; + }; }; }; + # runs on a merge request for example + pipelines."merge_request_event" = { + stages = ["some_stage"]; + jobs = { ... }; + }; }; ... } @@ -39,13 +56,28 @@ Also makes it possible to split CI parts in a separate module which can be impor } ``` +Now either use this in your .gitlab-ci.yml or setup Soonix to auto generate this +file for you with the right version (see the [docs][docs-soonix] for more). + ```yaml # .gitlab-ci.yml include: - - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@1.1.1 + - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: pin to the latest release/version (don't use "main" etc.) inputs: - # specify inputs here, for example: - image_tag: latest-cachix + version: # docker image tag, use the same version as a above +``` + +## Usage (directly) + +```nix +let + cilib = inputs.nix-gitlab-ci.lib {inherit pkgs;}; +in + cilib.mkCI { + config = ...; + pipelines."default" = ...; + }; + # exposes `soonix` for the soonix hook and `packages` which contain the configs, jobs etc. ``` ## Utilities @@ -55,24 +87,25 @@ include: To disable any of the provided caches for a pipeline one can set `NIX_CI_DISABLE_CACHE` to anything non-empty (eg. "yes") when triggering the pipeline. -The `build:nix-ci` job has a different special environment variable `NIX_CI_SKIP_CACHE` -(useful if the generated pipeline is outdated but caching should generally still take place). +The `build:nix-ci` job has a different special environment variable `NIX_CI_FORCE_BUILD` +(useful if the generated pipeline in the cache is outdated, this will build it again). ### Run Jobs locally You can run any job's script (+ before and after) locally with Nix for easier testing: ```sh -nix run .#gitlab-ci-job: +# / pipeline name, like "default" +nix run .#gitlab-ci:pipeline::job: ``` -There is also `.#gitlab-ci-job-deps:` which generates and exports the required environment variables for each job: +There is also `.#gitlab-ci:pipeline::job-deps:` which generates and exports the required environment variables for each job: - PATH (with all deps) - any custom env variables which contain store paths to not break stuff when switching archs -Please see #8 for some issues and further improvements on this. - ## Thanks to Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix + +[docs-soonix]: https://nix-gitlab-ci.projects.tf/soonix "Soonix Integration" diff --git a/docs/caching.md b/docs/caching.md new file mode 100644 index 0000000..0a84110 --- /dev/null +++ b/docs/caching.md @@ -0,0 +1,50 @@ +# Caching + +Nix GitLab CI supports several caching mechanisms to speed up your pipelines. + +## GitLab Runner Cache + +The runner cache strategy copies the new store paths into a directory `.nix-cache`, +which is then saved in the regular GitLab cache (technically runner cache). +It's also configured as a substituter automatically. + +To enable, set the cache strategy to `runner`. + +Configure it using these environment variables: + +- `RUNNER_CACHE`: path to the runner cache (default `.nix-cache`) + +!!! warning + + This is very inefficient and should probably only be used for very very small + dependency counts. Otherwise it takes an eternity to save to cache. + +## Cachix + +Cachix is a hosted binary cache service that can significantly speed up Nix +builds by sharing build results. + +To enable, set the cache strategy to `attic`. + +Configure it using these environment variables: + +- `CACHIX_CACHE`: name of the cache to use +- (`CACHIX_AUTH_TOKEN`): cachix client itself uses this for authentication + +!!! warning + + Cachix has not been tested. Feedback is appreciated :) + +## Attic (Self-Hosted Cache) + +Attic is a self-hosted, deduplicating binary cache. It's a great option if you +want more control over your caching infrastructure and to have the cache closer +to your runners. + +To enable, set the cache strategy to `attic`. + +Configure it using these environment variables: + +- `ATTIC_SERVER`: URL of the server +- `ATTIC_CACHE`: name of the cache to use +- `ATTIC_TOKEN`: auth token from the attic server diff --git a/docs/cicd_component.md b/docs/cicd_component.md new file mode 100644 index 0000000..4ea3b17 --- /dev/null +++ b/docs/cicd_component.md @@ -0,0 +1,48 @@ +# CI/CD Component + +The CI/CD Component has some inputs which configure defaults for Nix GitLab CI. + +!!! WARNING + + If you get errors like `the component path is not supported` it might be related to + [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/437996#note_1775337668). + + See [here](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/issues/27) for more. + +## `version` + +- Type: `string` + +Which version of the Nix CI image to use. Using a tag/version is recommended. +Will not do anything if a custom image is specified using `NIX_CI_IMAGE`. + +## `cache_strategy` + +- Type: `string` +- Default: `"auto"` +- Options: `auto` | `none` | `runner` | `cachix` | `attic` + +Sets the default caching strategy. + +- `auto`: dynamically selects the best strategy for every job based on env variables +- `none`: disables caching +- `runner`, `cachix` & `attic`: forces every job to use this strategy + +Can be overridden by `NIX_CI_CACHE_STRATEGY`, see [Environment Variables](./environment_variables.md). + +## `cache_files` + +- Type: `array` (of strings) +- Default: `["flake.nix", "flake.lock"]` + +Files to use as the cache key for the generated pipeline yaml. +If you use a file like `ci.nix` to define CI, add that here for example. +This makes sure that changes to your Nix CI configuration will invalidate the cache, +otherwise an old pipeline yaml might be used. + +!!! warning + + The value of this is used in `cache:key:files`, which currently only supports + a max of 2 entries. So use something like `["flake.*", "ci.nix"]` to match + `flake.lock`, `flake.nix` and `ci.nix`. + See [gitlab-org/gitlab#301161](https://gitlab.com/gitlab-org/gitlab/-/issues/301161) diff --git a/docs/environment_variables.md b/docs/environment_variables.md new file mode 100644 index 0000000..56c0ba5 --- /dev/null +++ b/docs/environment_variables.md @@ -0,0 +1,107 @@ +# Environment Variables + +Nix GitLab CI is mostly controlled using environment variables. +This page outlines all the variables and their use case. + +## `NIX_CI_IMAGE` + +| | | +| ----------- | -------------------------------------------------------------------------- | +| Default | `registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci@$[[ inputs.version ]]` | +| Description | Image to use for the jobs | + +## `NIX_CI_PIPELINE_NAME` + +| | | +| ----------- | ------------------------------------------------- | +| Default | N/A | +| Description | Explicitly request a pipeline to be built and ran | +| See also | [Multi Pipeline](./multi_pipeline.md) | + +## `NIX_CI_DEFAULT_SOURCES` + +| | | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| Default | `.*` | +| Description | Regex to match `$CI_PIPELINE_SOURCE` against. If it matches, the `default` pipeline will be ran, otherwise `$CI_PIPELINE_SOURCE` | +| See also | [Multi Pipeline](./multi_pipeline.md) | + +## `NIX_CI_FORCE_BUILD` + +| | | +| ----------- | -------------------------------------------------------------------------------------- | +| Default | N/A | +| Description | Set to any non-empty value to force the `nix-ci:build` job to freshly build the config | +| See also | [Caching](./caching.md) | + +## `NIX_CI_DISABLE_CACHE` + +| | | +| ----------- | ------------------------------------------------------ | +| Default | N/A | +| Description | Set to any non-empty value to disable caching for jobs | +| See also | [Caching](./caching.md) | + +## `NIX_CI_CACHE_STRATEGY` + +| | | +| ----------- | --------------------------------------------------------------------------------- | +| Default | `$[[ inputs.cache_strategy ]]` -> defaults to `auto` | +| Description | Caching strategy to use. `auto` will select the strategy based on runner settings | +| See also | [Caching](./caching.md) | + +## `NIX_CI_RUNNER_CACHE_STRATEGY` + +| | | +| ----------- | ---------------------------------------------------------------- | +| Default | N/A | +| Description | Every runner can set it's own preferred cache strategy with this | +| See also | [Caching](./caching.md) | + +## `NIX_CI_DEFAULT_CACHE_STRATEGY` + +| | | +| ----------- | ------------------------------------------------------------------------------------------------- | +| Default | `none` | +| Description | If no runner cache strategy is set and the main strategy is set to auto, this will be the default | +| See also | [Caching](./caching.md) | + +## `RUNNER_CACHE` + +| | | +| ----------- | -------------------------------------- | +| Default | `.nix-cache` | +| Description | Path to directory for the runner cache | +| See also | [Caching](./caching.md) | + +## `CACHIX_CACHE` + +| | | +| ----------- | ------------------------------- | +| Default | N/A | +| Description | Name of the cachix cache to use | +| See also | [Caching](./caching.md) | + +## `ATTIC_CACHE` + +| | | +| ----------- | ------------------------------ | +| Default | N/A | +| Description | Name of the attic cache to use | +| See also | [Caching](./caching.md) | + +## `ATTIC_SERVER` + +| | | +| ----------- | ----------------------- | +| Default | N/A | +| Description | URL of the attic server | +| See also | [Caching](./caching.md) | + +## `ATTIC_TOKEN` + +| | | +| ----------- | ------------------------------- | +| Default | N/A | +| Description | API token from the attic server | +| See also | [Caching](./caching.md) | diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..336a55a --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,13 @@ +# Example Configs + +- [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci) +- [TECHNOFAB/nixlets](https://gitlab.com/TECHNOFAB/nixlets) +- [TECHNOFAB/nixible](https://gitlab.com/TECHNOFAB/nixible) +- [TECHNOFAB/nixmkdocs](https://gitlab.com/TECHNOFAB/nixmkdocs) +- [TECHNOFAB/tofunix](https://gitlab.com/TECHNOFAB/tofunix) +- [TECHNOFAB/nixtest](https://gitlab.com/TECHNOFAB/nixtest) + +!!! note + + Feel free to edit this page and add your project if you're using + Nix GitLab CI :) diff --git a/docs/images/logo.svg b/docs/images/logo.svg new file mode 100755 index 0000000..d8dce95 --- /dev/null +++ b/docs/images/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..14f7da3 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,22 @@ +# Nix GitLab CI + +This project provides a Nix flake module that allows you to generate your `.gitlab-ci.yml` file directly from your Nix configuration. + +## Features + +- **Reproducibility:** Leverage Nix's strength in creating reproducible environments for your CI jobs. +- **Easy Dependency Management:** Easily include any package available in Nixpkgs or your own defined packages within your CI jobs using Nix. +- **Modularity:** Define and manage your CI configurations in a structured and modular way using Nix modules, making it easier to share and reuse CI logic across multiple projects. + +This documentation will guide you through setting up and using Nix GitLab CI for your projects. + +## Warnings + +To save you from frantically searching these docs if something doesn't work as expected, here are the most important warnings ;) + +!!! warning + + Do not put Nix store paths into global/pipeline variables. They will simply be passed through, + resulting in bad portability (if two runners have different archs for example, one cannot find the path). + If you need any Nix store path in env variables, always do it on the job level, there + it will automatically be computed at runtime, thus will always work no matter which runner it runs on. diff --git a/docs/kubernetes_runner.md b/docs/kubernetes_runner.md new file mode 100644 index 0000000..4c20421 --- /dev/null +++ b/docs/kubernetes_runner.md @@ -0,0 +1,38 @@ +# Kubernetes Runner Setup + +Using the GitLab Kubernetes runner allows your CI jobs to run as pods in a Kubernetes cluster. +Nix GitLab CI can be integrated with this setup, and using advanced configuration options like +`pod_spec` makes it easy to add runner specific caching. + +Using this Runner configuration ... + +```toml +[[runners.kubernetes.pod_spec]] +name = "nix-ci-cache-secrets" +patch = ''' + containers: + - name: build + envFrom: + - secretRef: + name: nix-ci-cache-env +''' +``` + +... and a secret containing ... + +```yaml +NIX_CI_RUNNER_CACHE_STRATEGY: attic +ATTIC_SERVER: # example: http://atticd..svc.cluster.local:8080 +ATTIC_CACHE: ci # name however you want, just needs to exist +ATTIC_TOKEN: +``` + +... makes your jobs automatically cache their Nix store paths to the in-cluster +attic when running with this runner. + +Other runners could use cachix or no cache, you get the idea ;P + +!!! note + + This of course works with any executor where you can set environment + variables. This is just an example how to do it in Kubernetes easily. diff --git a/docs/multi_pipeline.md b/docs/multi_pipeline.md new file mode 100644 index 0000000..d8dbfe6 --- /dev/null +++ b/docs/multi_pipeline.md @@ -0,0 +1,31 @@ +# Multiple Pipelines + +With V2, Nix GitLab CI can generate different pipelines, depending on the +pipeline source (`$CI_PIPELINE_SOURCE`). + +By default, no matter which source, the `default` pipeline is built and ran. +`$NIX_CI_PIPELINE` can override that, eg. when manually triggering a run. +To configure which source should be 1-to-1 translated to a pipeline with the +same name, set `$NIX_CI_DEFAULT_SOURCES` to a regex which explicitly does not +match these sources. Or set it to an impossible to match regex, then it will +always run the pipeline named after `$CI_PIPELINE_SOURCE`. + +## Example 1: always run default + +If you only have a single pipeline, you just have to call it `default`. +Everything else works out of the box. + +## Example 2: default and merge_request_event + +If you want the source `merge_request_event` to trigger a different pipeline, +name it like that and set `$NIX_CI_DEFAULT_SOURCES` to `^(merge_request_event)$`. +Now a merge request will run this pipeline, while everything else runs `default`. + +## Example 3: default, push and web + +Set `$NIX_CI_DEFAULT_SOURCES` to `^(push|web)$`. + +## Example 4: always run the specific pipelines, never default + +Set `$NIX_CI_DEFAULT_SOURCES` to any regex that never matches the sources, +like `a\A` or `nothing`. diff --git a/docs/options.md b/docs/options.md new file mode 100644 index 0000000..4ca74a4 --- /dev/null +++ b/docs/options.md @@ -0,0 +1,3 @@ +# Options + +{% include 'options.md' %} diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..c1f85df --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,81 @@ +# Setup + +To integrate Nix GitLab CI into your project, you need to make two main changes: + +1. Add the `nix-gitlab-ci` flake module to your `flake.nix`. +1. Include the necessary component in your `.gitlab-ci.yml`. + +## Adding to `flake.nix` + +In your project's `flake.nix`, add `nix-gitlab-ci` as an input and import its +flake module within your `flake-parts` configuration. + +```nix title="flake.nix" +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; # Or your preferred nixpkgs branch/version + flake-parts.url = "github:hercules-ci/flake-parts"; + + # Add nix-gitlab-ci as an input + # recommendation: pin to a specific release/version + nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; + }; + + outputs = { nixpkgs, flake-parts, ... }@inputs: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + # Import the nix-gitlab-ci flake module + inputs.nix-gitlab-ci.flakeModule + ]; + + systems = [ + "x86_64-linux" + "aarch64-linux" + # Add other systems you need + ]; + + perSystem = { pkgs, ... }: { + # define your CI pipelines + # ci.pipelines."merge_request_event" = { ... }; + }; + }; +} +``` + +Replace `` with the specific version or commit hash of `nix-gitlab-ci` +you wish to use. Pinning to a specific version is highly recommended for +reproducibility and compatibility. + +!!! warning + + While the flake input is locked through `flake.lock`, the CI/CD component + will always use the latest commit of the reference. This means that by using + a branch like `main` as version for both, the CI/CD component will always use + the latest commit while your flake uses a fixed one. + This could result in drift between both, potentially breaking stuff. + +## Including in `.gitlab-ci.yml` + +Your `.gitlab-ci.yml` file will be minimal. Its primary role is to include the +`nix-gitlab-ci` component, which will then generate the full CI configuration +based on your Nix code. + +```yaml title=".gitlab-ci.yml" +include: + - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ + inputs: + # This input sets the Docker image tag used for the CI jobs. + # Use the same version as you pinned in your flake.nix for consistency. + version: +``` + +Again, ensure `` matches the version used in your `flake.nix`. +This component includes a job (`build:nix-ci`) that will evaluate your Nix +configuration and generate the `.gitlab-ci.yml` used for the pipeline run. + +!!! note + + Since V3 [Soonix](https://soonix.projects.tf) is supported, this can + automatically generate the `.gitlab-ci.yml` for you, with the version + automatically following the flake. + See [Soonix Integration](./soonix.md) for more. diff --git a/docs/soonix.md b/docs/soonix.md new file mode 100644 index 0000000..b333ab6 --- /dev/null +++ b/docs/soonix.md @@ -0,0 +1,20 @@ +# Soonix Integration + +[Soonix](https://soonix.projects.tf) can be used to automatically generate the +`.gitlab-ci.yml` for you. +This will by default include the CI/CD component with the same version as the +flake (using the `VERSION` file in this repo). + +You can specify some options to configure this, like changing the component URL +or adding extra data to your `.gitlab-ci.yml` (like this repo does to bootstrap +Nix-GitLab-CI). See [Options](./options.md#configsoonix) for all the config options. + +You can use it like this: + +```nix +let + ci = cilib.mkCI { ... }; +in { + soonix.hooks."ci" = ci.soonix; +} +``` diff --git a/docs/style.css b/docs/style.css new file mode 100644 index 0000000..b2ae4ff --- /dev/null +++ b/docs/style.css @@ -0,0 +1,15 @@ +.md-header__button.md-logo { + margin: 0; + padding-top: .2rem; + padding-bottom: .2rem; +} + +[dir="ltr"] .md-header__title { + margin-left: 0; +} + +.md-header__button.md-logo img, +.md-header__button.md-logo svg { + height: 2rem; +} + diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..44c0bf7 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,73 @@ +# Usage + +## Usage (with flake-parts) + +```nix +# flake.nix +{ + ... + inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; # recommendation: pin to the latest release/version + + outputs = {...}: flake-parts.lib.mkFlake {...} { + imports = [ + inputs.nix-gitlab-ci.flakeModule + ]; + ... + + perSystem = {pkgs, ...}: { + ci = { + config = { + # configure Nix-GitLab-CI here, see docs for options + }; + pipelines."default" = { + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + nix.deps = [pkgs.unixtools.ping]; + script = [ + "ping -c 5 8.8.8.8" + ]; + }; + }; + }; + # runs on a merge request for example + pipelines."merge_request_event" = { + stages = ["some_stage"]; + jobs = { ... }; + }; + }; + ... + } + } +} +``` + +Now either use this in your .gitlab-ci.yml or setup Soonix to auto generate this +file for you with the right version (see the [docs][docs-soonix] for more). + +```yaml +# .gitlab-ci.yml +include: + - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: pin to the latest release/version (don't use "main" etc.) + inputs: + version: # docker image tag, use the same version as a above +``` + +## Usage (directly) + +```nix +let + cilib = inputs.nix-gitlab-ci.lib {inherit pkgs;}; +in + cilib.mkCI { + config = ...; + pipelines."default" = ...; + }; + # exposes `soonix` for the soonix hook and `packages` which contain the configs, jobs etc. +``` + +______________________________________________________________________ + +Since V2 multiple pipelines are supported. +See [Multiple Pipelines](./multi_pipeline.md) for more. diff --git a/docs/utilities.md b/docs/utilities.md new file mode 100644 index 0000000..b82091d --- /dev/null +++ b/docs/utilities.md @@ -0,0 +1,64 @@ +# Utilities + +Nix GitLab CI provides a couple of utilities to help with development and +debugging. + +## Disabling caching temporarily + +Nix GitLab CI often utilizes caching mechanisms to speed up your pipelines +(see [Caching](./caching.md)). +However, there might be situations where you need to temporarily disable these +caches for a specific pipeline run, for example, to debug a caching issue or +ensure a clean build. + +To disable most of the provided caches for a pipeline, set the environment +variable `NIX_CI_DISABLE_CACHE` to any non-empty value (e.g., `yes`, `true`, `1`) +when triggering the pipeline in the GitLab UI or via the API. + +## Forcing a rebuild of the CI pipeline definition + +The job responsible for generating the `.gitlab-ci.yml` from your Nix code +(`build:nix-ci`) might itself be cached. If you've made changes to your Nix CI +configuration and the pipeline doesn't seem to pick them up, the cached job +definition might be the reason. + +You should first double check if all the Nix files you defined the CI config in +are specified in the `cache_files` CI/CD-component input +(see [CI/CD Component](./cicd_component.md) for more). + +To force this specific job to rebuild and re-evaluate your Nix configuration, +set the environment variable `NIX_CI_FORCE_BUILD` when triggering the pipeline. + +## Running jobs locally + +One of the benefits of defining your CI jobs with Nix is the ability to run them +locally in an environment that closely mirrors the CI environment. This can +significantly speed up debugging and development. + +You can run the script of any defined job locally using the `nix run` command. +The syntax is: + +```sh +nix run .#gitlab-ci:pipeline::job: +``` + +Replace `` with the name of the pipeline the job belongs to +(e.g., `default` for jobs defined under the `ci` attribute) and `` +with the name of the job you want to run. + +This command will set up the environment with the specified `nix.deps` and +execute the job's `script`. + +There is also an attribute `.#gitlab-ci:pipeline::job-deps:`. +Building this derivation will generate a shell script which exports the required +environment variables for the job, such as the `PATH` including all dependencies +and any custom environment variables that contain store paths (ensuring they are +correctly resolved across different architectures). + +You can use this to inspect the environment that would be set up for a job without +running the full script. + +## Viewing generated config in YAML + +Since the GitLab CI config is generated simply using JSON, it's hard to read and +debug. For debugging V3 now adds another package `gitlab-ci:pipeline::pretty`. diff --git a/examples/flake-parts/flake.lock b/examples/flake-parts/flake.lock new file mode 100644 index 0000000..2e81bcf --- /dev/null +++ b/examples/flake-parts/flake.lock @@ -0,0 +1,95 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1756770412, + "narHash": "sha256-+uWLQZccFHwqpGqr2Yt5VsW/PbeJVTn9Dk6SHWhNRPw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "4524271976b625a4a605beefd893f270620fd751", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nix-gitlab-ci": { + "locked": { + "dir": "lib", + "lastModified": 1752052838, + "narHash": "sha256-EqP4xB8YTVXWPCCchnVtQbuq0bKa79TUEcPF3hjuX/k=", + "owner": "TECHNOFAB", + "repo": "nix-gitlab-ci", + "rev": "0c6949f585a2c1ea2cf85fc01445496f7c75faae", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "TECHNOFAB", + "repo": "nix-gitlab-ci", + "type": "gitlab" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1756696532, + "narHash": "sha256-6FWagzm0b7I/IGigOv9pr6LL7NQ86mextfE8g8Q6HBg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "58dcbf1ec551914c3756c267b8b9c8c86baa1b2f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1754788789, + "narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "a73b9c743612e4244d865a2fdee11865283c04e6", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nix-gitlab-ci": "nix-gitlab-ci", + "nixpkgs": "nixpkgs", + "systems": "systems" + } + }, + "systems": { + "locked": { + "lastModified": 1689347949, + "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", + "owner": "nix-systems", + "repo": "default-linux", + "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default-linux", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/flake-parts/flake.nix b/examples/flake-parts/flake.nix new file mode 100644 index 0000000..c1b7977 --- /dev/null +++ b/examples/flake-parts/flake.nix @@ -0,0 +1,39 @@ +{ + outputs = { + flake-parts, + systems, + ... + } @ inputs: + flake-parts.lib.mkFlake {inherit inputs;} { + imports = [ + inputs.nix-gitlab-ci.flakeModule + ]; + systems = import systems; + flake = {}; + perSystem = _: { + ci = { + pipelines = { + "default" = { + stages = ["example"]; + jobs."example" = { + stage = "example"; + script = ["echo hello world"]; + }; + }; + "test".jobs."example" = { + stage = ".pre"; + script = ["echo hello world"]; + }; + }; + }; + }; + }; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + systems.url = "github:nix-systems/default-linux"; + # NOTE: better pin to a version + nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; + }; +} diff --git a/examples/rensa-nix/flake.lock b/examples/rensa-nix/flake.lock new file mode 100644 index 0000000..7c47f5c --- /dev/null +++ b/examples/rensa-nix/flake.lock @@ -0,0 +1,63 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1756542300, + "narHash": "sha256-tlOn88coG5fzdyqz6R93SQL5Gpq+m/DsWpekNFhqPQk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d7600c775f877cd87b4f5a831c28aa94137377aa", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1754184128, + "narHash": "sha256-AjhoyBL4eSyXf01Bmc6DiuaMrJRNdWopmdnMY0Pa/M0=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "02e72200e6d56494f4a7c0da8118760736e41b60", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "ren": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "dir": "lib", + "lastModified": 1756370106, + "narHash": "sha256-l84ojcHuQWBwn4BRxQsMMfQpcq/Az/sHh/hSqFgVtyg=", + "owner": "rensa-nix", + "repo": "core", + "rev": "9c1a29fa9ba7cbbb78b9e47eb8afbcd29303a3b4", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "rensa-nix", + "repo": "core", + "type": "gitlab" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "ren": "ren" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/rensa-nix/flake.nix b/examples/rensa-nix/flake.nix new file mode 100644 index 0000000..1f81895 --- /dev/null +++ b/examples/rensa-nix/flake.nix @@ -0,0 +1,30 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + ren.url = "gitlab:rensa-nix/core?dir=lib"; + }; + + outputs = { + self, + ren, + ... + } @ inputs: + ren.buildWith + { + inherit inputs; + cellsFrom = ./nix; + transformInputs = system: i: + i + // { + pkgs = import i.nixpkgs {inherit system;}; + }; + cellBlocks = with ren.blocks; [ + (simple "ci") + ]; + } + { + packages = ren.select self [ + ["repo" "ci" "packages"] + ]; + }; +} diff --git a/examples/rensa-nix/nix/repo/ci.nix b/examples/rensa-nix/nix/repo/ci.nix new file mode 100644 index 0000000..adb707e --- /dev/null +++ b/examples/rensa-nix/nix/repo/ci.nix @@ -0,0 +1,18 @@ +{inputs, ...}: let + inherit (inputs) cilib; +in + cilib.mkCI { + pipelines = { + "default" = { + stages = ["example"]; + jobs."example" = { + stage = "example"; + script = ["echo hello world"]; + }; + }; + "test".jobs."example" = { + stage = ".pre"; + script = ["echo hello world"]; + }; + }; + } diff --git a/examples/rensa-nix/nix/repo/flake.lock b/examples/rensa-nix/nix/repo/flake.lock new file mode 100644 index 0000000..fb28eb7 --- /dev/null +++ b/examples/rensa-nix/nix/repo/flake.lock @@ -0,0 +1,28 @@ +{ + "nodes": { + "nix-gitlab-ci-lib": { + "locked": { + "dir": "lib", + "lastModified": 1752052838, + "narHash": "sha256-EqP4xB8YTVXWPCCchnVtQbuq0bKa79TUEcPF3hjuX/k=", + "owner": "TECHNOFAB", + "repo": "nix-gitlab-ci", + "rev": "0c6949f585a2c1ea2cf85fc01445496f7c75faae", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "TECHNOFAB", + "repo": "nix-gitlab-ci", + "type": "gitlab" + } + }, + "root": { + "inputs": { + "nix-gitlab-ci-lib": "nix-gitlab-ci-lib" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/examples/rensa-nix/nix/repo/flake.nix b/examples/rensa-nix/nix/repo/flake.nix new file mode 100644 index 0000000..ad3b77d --- /dev/null +++ b/examples/rensa-nix/nix/repo/flake.nix @@ -0,0 +1,10 @@ +{ + inputs = { + nix-gitlab-ci-lib.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; + }; + outputs = i: + i + // { + cilib = i.nix-gitlab-ci-lib.lib {inherit (i.parent) pkgs;}; + }; +} diff --git a/flake.lock b/flake.lock index dd864f4..60df9c4 100644 --- a/flake.lock +++ b/flake.lock @@ -1,227 +1,12 @@ { "nodes": { - "cachix": { - "inputs": { - "devenv": [ - "devenv" - ], - "flake-compat": [ - "devenv" - ], - "git-hooks": [ - "devenv" - ], - "nixpkgs": "nixpkgs" - }, - "locked": { - "lastModified": 1728672398, - "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", - "owner": "cachix", - "repo": "cachix", - "rev": "aac51f698309fd0f381149214b7eee213c66ef0a", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "latest", - "repo": "cachix", - "type": "github" - } - }, - "devenv": { - "inputs": { - "cachix": "cachix", - "flake-compat": "flake-compat", - "git-hooks": [ - "git-hooks" - ], - "nix": "nix", - "nixpkgs": "nixpkgs_3" - }, - "locked": { - "lastModified": 1732585607, - "narHash": "sha256-6ffeaSMuaL326f7KrCeScpSJtdHsFKS9gPrsSZkndvU=", - "owner": "cachix", - "repo": "devenv", - "rev": "a520f05c40ebecaf5e17064b27e28ba8e70c49fb", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": [ - "devenv", - "nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1712014858, - "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "flake-parts_2": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "lastModified": 1730504689, - "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "506278e768c2a08bec68eb62932193e341f55c90", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "git-hooks": { - "inputs": { - "flake-compat": "flake-compat_2", - "gitignore": "gitignore", - "nixpkgs": "nixpkgs_4", - "nixpkgs-stable": "nixpkgs-stable" - }, - "locked": { - "lastModified": 1732021966, - "narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "3308484d1a443fc5bc92012435d79e80458fe43c", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "git-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "libgit2": { - "flake": false, - "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", - "owner": "libgit2", - "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", - "type": "github" - }, - "original": { - "owner": "libgit2", - "repo": "libgit2", - "type": "github" - } - }, - "nix": { - "inputs": { - "flake-compat": [ - "devenv" - ], - "flake-parts": "flake-parts", - "libgit2": "libgit2", - "nixpkgs": "nixpkgs_2", - "nixpkgs-23-11": [ - "devenv" - ], - "nixpkgs-regression": [ - "devenv" - ], - "pre-commit-hooks": [ - "devenv" - ] - }, - "locked": { - "lastModified": 1727438425, - "narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=", - "owner": "domenkozar", - "repo": "nix", - "rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546", - "type": "github" - }, - "original": { - "owner": "domenkozar", - "ref": "devenv-2.24", - "repo": "nix", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1730531603, - "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", + "lastModified": 1764667669, + "narHash": "sha256-7WUCZfmqLAssbDqwg9cUDAXrSoXN79eEEq17qhTNM/Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", + "rev": "418468ac9527e799809c900eda37cbff999199b6", "type": "github" }, "original": { @@ -233,153 +18,43 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1730504152, - "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" - }, - "original": { - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" - } - }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1730741070, - "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", + "lastModified": 1754184128, + "narHash": "sha256-AjhoyBL4eSyXf01Bmc6DiuaMrJRNdWopmdnMY0Pa/M0=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "02e72200e6d56494f4a7c0da8118760736e41b60", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixos-24.05", - "repo": "nixpkgs", + "owner": "nix-community", + "repo": "nixpkgs.lib", "type": "github" } }, - "nixpkgs_2": { + "ren": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, "locked": { - "lastModified": 1717432640, - "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870", - "type": "github" + "dir": "lib", + "lastModified": 1758738378, + "narHash": "sha256-NjzqdvQCDDdObEBH8x/vdhbdhrIB+N9E570uCdksGHY=", + "owner": "rensa-nix", + "repo": "core", + "rev": "abe19f9f13aff41de2b63304545c87d193d19ef4", + "type": "gitlab" }, "original": { - "owner": "NixOS", - "ref": "release-24.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_3": { - "locked": { - "lastModified": 1716977621, - "narHash": "sha256-Q1UQzYcMJH4RscmpTkjlgqQDX5yi1tZL0O345Ri6vXQ=", - "owner": "cachix", - "repo": "devenv-nixpkgs", - "rev": "4267e705586473d3e5c8d50299e71503f16a6fb6", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "rolling", - "repo": "devenv-nixpkgs", - "type": "github" - } - }, - "nixpkgs_4": { - "locked": { - "lastModified": 1730768919, - "narHash": "sha256-8AKquNnnSaJRXZxc5YmF/WfmxiHX6MMZZasRP6RRQkE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a04d33c0c3f1a59a2c1cb0c6e34cd24500e5a1dc", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_5": { - "locked": { - "lastModified": 1732238832, - "narHash": "sha256-sQxuJm8rHY20xq6Ah+GwIUkF95tWjGRd1X8xF+Pkk38=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "8edf06bea5bcbee082df1b7369ff973b91618b8d", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_6": { - "locked": { - "lastModified": 1731890469, - "narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "5083ec887760adfe12af64830a66807423a859a7", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" + "dir": "lib", + "owner": "rensa-nix", + "repo": "core", + "type": "gitlab" } }, "root": { "inputs": { - "devenv": "devenv", - "flake-parts": "flake-parts_2", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_5", - "systems": "systems", - "treefmt-nix": "treefmt-nix" - } - }, - "systems": { - "locked": { - "lastModified": 1689347949, - "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", - "owner": "nix-systems", - "repo": "default-linux", - "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default-linux", - "type": "github" - } - }, - "treefmt-nix": { - "inputs": { - "nixpkgs": "nixpkgs_6" - }, - "locked": { - "lastModified": 1732643199, - "narHash": "sha256-uI7TXEb231o8dkwB5AUCecx3AQtosRmL6hKgnckvjps=", - "owner": "numtide", - "repo": "treefmt-nix", - "rev": "84637a7ab04179bdc42aa8fd0af1909fba76ad0c", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "treefmt-nix", - "type": "github" + "nixpkgs": "nixpkgs", + "ren": "ren" } } }, diff --git a/flake.nix b/flake.nix index 596b693..769a6b9 100644 --- a/flake.nix +++ b/flake.nix @@ -1,230 +1,37 @@ { + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + ren.url = "gitlab:rensa-nix/core?dir=lib"; + }; + outputs = { - flake-parts, - systems, + self, + ren, ... } @ inputs: - flake-parts.lib.mkFlake {inherit inputs;} { - imports = [ - inputs.devenv.flakeModule - inputs.treefmt-nix.flakeModule - ./lib/flakeModule.nix + ren.buildWith + { + inherit inputs; + cellsFrom = ./nix; + transformInputs = system: i: + i + // { + pkgs = import i.nixpkgs {inherit system;}; + }; + cellBlocks = with ren.blocks; [ + (simple "devShells") + (simple "pkgs") + (simple "tests") + (simple "docs") + (simple "ci") + ]; + } + { + packages = ren.select self [ + ["repo" "tests"] + ["repo" "docs"] + ["repo" "ci" "packages"] + ["packages" "pkgs"] ]; - systems = import systems; - flake = {}; - perSystem = { - pkgs, - inputs', - config, - ... - }: rec { - treefmt = { - projectRootFile = "flake.nix"; - programs = { - alejandra.enable = true; - mdformat.enable = true; - yamlfmt.enable = true; - }; - }; - devenv.shells.default = { - containers = pkgs.lib.mkForce {}; - packages = with pkgs; [dive skopeo]; - - pre-commit = { - hooks = { - treefmt = { - enable = true; - packageOverrides.treefmt = config.treefmt.build.wrapper; - }; - }; - }; - }; - ci = { - # use the image built in the parent pipeline for dogfooding - config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA"; - stages = ["test"]; - jobs = { - "test" = { - stage = "test"; - nix = { - deps = [pkgs.hello pkgs.curl]; - disable-cache = false; - }; - variables = { - TEST = "test"; - TEST_WITH_DERIVATION = "${pkgs.hello}/test"; - }; - script = [ - "hello" - "curl google.de" - "echo $TEST $TEST_WITH_DERIVATION" - ]; - }; - "test-non-nix" = { - nix.enable = false; - stage = "test"; - image = "alpine:latest"; - script = [ - "echo \"This job will not be modified to use nix\"" - ]; - }; - }; - }; - - packages = let - setupScript = extra_setup: - pkgs.writeShellScriptBin "setup_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" - nix path-info --all > /tmp/nix-store-before - - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${extra_setup} - else - echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration" - fi - - export NIX_CONFIG=" - extra-trusted-public-keys = $NIX_PUBLIC_KEYS - extra-trusted-substituters = $NIX_SUBSTITUTERS - extra-substituters = $NIX_SUBSTITUTERS - $NIX_CONFIG - $NIX_EXTRA_CONFIG - " - echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" - - # load the job's deps only if the name was passed - if [[ ! -z $1 ]]; then - echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job" - nix build .#gitlab-ci-job-deps:$1 - source $(readlink -f result) - echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" - fi - ''; - finalizeScript = push_command: - pkgs.writeShellScriptBin "finalize_nix_ci" '' - echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache" - nix path-info --all > /tmp/nix-store-after - ${pkgs.diffutils}/bin/diff --new-line-format="%L" \ - --old-line-format="" --unchanged-line-format="" \ - /tmp/nix-store-before /tmp/nix-store-after \ - | { - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - ${push_command} - else - ${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; } - fi - } - echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" - ''; - mkImage = extraPackages: - pkgs.dockerTools.buildImage { - name = "nix-gitlab-ci"; - fromImage = pkgs.dockerTools.pullImage { - imageName = "nixpkgs/nix-flakes"; - imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; - sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; - finalImageName = "nixpkgs/nix-flakes"; - finalImageTag = "latest"; - }; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = - [ - pkgs.gitMinimal - pkgs.gnugrep - ] - ++ extraPackages; - pathsToLink = ["/bin"]; - }; - }; - in { - setup-script = - setupScript - # sh - '' - # extra_setup - true - ''; - finalize-script = - finalizeScript - # sh - '' - # push_command - true - ''; - image = mkImage [ - (setupScript - # sh - '' - cachedir="$(pwd)/.nix-cache" - echo "Configuring caching with the Runner Cache in $cachedir..." - export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true" - '') - (finalizeScript - # sh - '' - # add ^* to all store paths ending in .drv (prevent warning log spam) - ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true - '') - ]; - image-cachix = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with cachix..." - ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true - '') - (finalizeScript - # sh - '' - ${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true - '') - ]; - image-attic = mkImage [ - (setupScript - # sh - '' - echo "Configuring caching with attic..." - ${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true - ${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true - '') - (finalizeScript - # sh - '' - ${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true - '') - ]; - }; - - checks = packages; - }; }; - - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - - # flake & devenv related - flake-parts.url = "github:hercules-ci/flake-parts"; - systems.url = "github:nix-systems/default-linux"; - devenv = { - url = "github:cachix/devenv"; - inputs.git-hooks.follows = "git-hooks"; - }; - git-hooks.url = "github:cachix/git-hooks.nix"; - treefmt-nix.url = "github:numtide/treefmt-nix"; - }; - - nixConfig = { - extra-substituters = [ - "https://cache.nixos.org/" - "https://nix-community.cachix.org" - "https://devenv.cachix.org" - ]; - - extra-trusted-public-keys = [ - "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" - "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" - ]; - }; } diff --git a/gitlab-ci.yml b/gitlab-ci.yml deleted file mode 100644 index f1cb469..0000000 --- a/gitlab-ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# NOTE: DEPRECATED: please switch to the CI/CD Component or include "templates/nix-gitlab-ci.yml" -# -variables: - # latest | latest-cachix | latest-attic etc. - NIX_CI_IMAGE_TAG: latest -stages: - - build - - trigger -nix-ci:build: - stage: build - image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${NIX_CI_IMAGE_TAG} - before_script: - - source setup_nix_ci - script: - # build the generated-gitlab-ci.yml - - nix build .#gitlab-ci-config - - install result generated-gitlab-ci.yml - after_script: - # upload to binary cache - - finalize_nix_ci - artifacts: - paths: - - generated-gitlab-ci.yml -nix-ci:trigger: - stage: trigger - needs: - - nix-ci:build - trigger: - include: - - artifact: generated-gitlab-ci.yml - job: nix-ci:build - strategy: depend diff --git a/lib/VERSION b/lib/VERSION new file mode 100644 index 0000000..ef538c2 --- /dev/null +++ b/lib/VERSION @@ -0,0 +1 @@ +3.1.2 diff --git a/lib/default.nix b/lib/default.nix new file mode 100644 index 0000000..ff3ec88 --- /dev/null +++ b/lib/default.nix @@ -0,0 +1,29 @@ +args: let + # allow passing just pkgs aswell for convenience + lib = args.lib or args.pkgs.lib; + # makes it optional to pass if it's not explicitly needed + pkgs = args.pkgs or (throw "[nix-gitlab-ci] pkgs argument was used but not set, please pass it"); + inherit (lib) evalModules trimWith; + + impl = import ./impl {inherit lib pkgs cilib;}; + + cilib = { + inherit (impl) helpers modules mkPipeline mkJobRun mkJobDeps mkJobPatched; + utils = import ./utils.nix {inherit pkgs;}; + version = trimWith { + start = true; + end = true; + } (builtins.readFile ./VERSION); + + mkCI = config: + (evalModules { + modules = [ + cilib.modules.nixCiSubmodule + { + inherit config; + } + ]; + }).config; + }; +in + cilib diff --git a/lib/flake.nix b/lib/flake.nix index 5d00bb6..227f9a6 100644 --- a/lib/flake.nix +++ b/lib/flake.nix @@ -1,9 +1,6 @@ { - description = "Nix-CI lib"; - - outputs = {...} @ inputs: - { - flakeModule = import ./flakeModule.nix; - } - // (import ./utils.nix); + outputs = _i: { + lib = import ./.; + flakeModule = ./flakeModule.nix; + }; } diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index 64f7926..aadf292 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,260 +9,16 @@ pkgs, ... }: let - cfg = config.ci.config; - - filterAttrsRec = pred: v: - if lib.isAttrs v - then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v) - else v; - - subType = options: lib.types.submodule {inherit options;}; - mkNullOption = type: - lib.mkOption { - default = null; - type = lib.types.nullOr type; - }; - - configType = with lib; - subType { - default-nix-image = mkOption { - type = types.str; - default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest"; - description = "The image to use on nix jobs"; - }; - nix-jobs-per-default = mkOption { - type = types.bool; - default = true; - description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; - }; - disable-cache = mkOption { - type = types.bool; - default = false; - description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; - description = "Cache key to use for the nix cache"; - }; - }; - jobType = with lib; - subType { - # nix ci opts - nix = mkOption { - type = subType { - enable = mkOption { - type = types.bool; - default = cfg.nix-jobs-per-default; - description = "Handle this job as a nix job"; - }; - deps = mkOption { - type = types.listOf types.package; - default = []; - description = "Dependencies/packages to install for this job"; - }; - disable-cache = mkOption { - type = types.bool; - default = cfg.disable-cache; - description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE"; - }; - cache-key = mkOption { - type = types.str; - default = cfg.cache-key; - description = "Cache key to use for the nix cache"; - }; - }; - default = {}; - description = "Configure Nix Gitlab CI for each job individually"; - }; - # gitlab opts - script = mkOption { - type = types.listOf types.str; - default = []; - }; - stage = mkOption { - type = types.str; - default = "test"; - }; - image = mkOption { - type = types.str; - default = cfg.default-nix-image; - }; - after_script = mkNullOption (types.listOf types.str); - allow_failure = mkNullOption (types.either types.attrs types.bool); - artifacts = mkNullOption (types.attrs); - before_script = mkNullOption (types.listOf types.str); - cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); - coverage = mkNullOption (types.str); - dependencies = mkNullOption (types.listOf types.str); - environment = mkNullOption (types.either types.attrs types.str); - extends = mkNullOption (types.str); - hooks = mkNullOption (types.attrs); - id_tokens = mkNullOption (types.attrs); - "inherit" = mkNullOption (types.attrs); - interruptible = mkNullOption (types.bool); - needs = mkNullOption (types.listOf (types.either types.str types.attrs)); - publish = mkNullOption (types.str); - pages = mkNullOption (types.attrs); - parallel = mkNullOption (types.either types.int types.attrs); - release = mkNullOption (types.attrs); - retry = mkNullOption (types.either types.int types.attrs); - rules = mkNullOption (types.listOf types.attrs); - resource_group = mkNullOption (types.str); - secrets = mkNullOption (types.attrs); - services = mkNullOption (types.listOf types.attrs); - start_in = mkNullOption (types.str); - tags = mkNullOption (types.listOf types.str); - timeout = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - when = mkNullOption (types.str); - }; + cilib = import ./. {inherit lib pkgs;}; + inherit (lib) types mkOption; in { - options = with lib; { + options = { ci = mkOption { - type = subType { - config = mkOption { - type = configType; - description = '' - Configuration options for the nix part itself - ''; - default = {}; - }; - image = mkNullOption (types.str); - variables = mkNullOption (types.attrs); - default = mkNullOption (types.attrs); - stages = mkNullOption (types.listOf types.str); - include = mkNullOption (types.attrs); - workflow = mkNullOption (types.attrs); - jobs = mkOption { - type = types.lazyAttrsOf jobType; - default = {}; - }; - }; - description = '' - Generate a Gitlab CI configuration which can be used to trigger a child pipeline. - This will inject code which pre-downloads the nix deps before each job and adds them to PATH. - ''; + type = types.submodule cilib.modules.nixCiSubmodule; default = {}; }; }; - - config.legacyPackages = let - toYaml = (pkgs.formats.yaml {}).generate; - mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); - prepend = key: arr: job: - job - // lib.optionalAttrs job.nix.enable { - ${key} = - arr - ++ (job.${key} or []); - }; - append = key: arr: job: - job - // lib.optionalAttrs job.nix.enable { - ${key} = (job.${key} or []) ++ arr; - }; - prependToBeforeScript = prepend "before_script"; - appendToAfterScript = append "after_script"; - - # filter job's variables to either only those containing store paths - # or those that do not - filterJobVariables = nix: job: - lib.concatMapAttrs ( - name: value: - lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { - ${name} = value; - } - ) - (job.variables or {}); - - jobs = filterAttrsRec (n: v: v != null) config.ci.jobs; - rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job-deps:${key}"; - value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' - export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; - ${variableExports} - ''; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - mapAttrs (key: job: let - variablesWithStorePaths = filterJobVariables false job; - variableExports = lib.concatLines ( - lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - in { - name = "gitlab-ci-job:${key}"; - value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - # set up deps and environment variables containing store paths - . ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - ${lib.concatLines (job.before_script or [])} - echo -e "\e[32mRunning script...\e[0m" - ${lib.concatLines job.script} - echo -e "\e[32mRunning after_script...\e[0m" - ${lib.concatLines (job.after_script or [])} - ''; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - mapAttrs (key: job: { - name = key; - value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - builtins.removeAttrs ( - (prependToBeforeScript [ - "source setup_nix_ci ${key}" - ] - (appendToAfterScript [ - "finalize_nix_ci" - ] - job)) - // lib.optionalAttrs job.nix.enable { - image = job.image; - variables = - (filterJobVariables false job) - // lib.optionalAttrs job.nix.disable-cache { - NIX_CI_DISABLE_CACHE = "yes"; - }; - cache = - ( - let - c = job.cache or []; - in - if builtins.isList c - then c - else [c] - ) - ++ (lib.optional (!job.nix.disable-cache) { - key = job.nix.cache-key; - paths = [".nix-cache/"]; - }); - } - ) ["nix"]; - }) - jobs; - in - { - gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript; + config.legacyPackages = config.ci.packages; } ); } diff --git a/lib/impl/default.nix b/lib/impl/default.nix new file mode 100644 index 0000000..7ff5623 --- /dev/null +++ b/lib/impl/default.nix @@ -0,0 +1,12 @@ +{ + pkgs, + lib, + cilib, +}: rec { + helpers = import ./helpers.nix {inherit lib pkgs;}; + mkJobDeps = import ./jobDeps.nix {inherit lib helpers;}; + mkJobRun = import ./jobRun.nix {inherit lib pkgs helpers;}; + mkJobPatched = import ./jobPatched.nix {inherit lib helpers;}; + mkPipeline = import ./pipeline.nix {inherit lib helpers mkJobDeps mkJobRun mkJobPatched;}; + modules = import ./modules {inherit lib cilib;}; +} diff --git a/lib/impl/helpers.nix b/lib/impl/helpers.nix new file mode 100644 index 0000000..754cead --- /dev/null +++ b/lib/impl/helpers.nix @@ -0,0 +1,123 @@ +{ + pkgs, + lib, +} @ args: let + inherit + (lib) + types + isAttrs + filterAttrs + mapAttrs + mkOption + mkOptionType + isType + literalExpression + pipe + hasInfix + concatMapAttrs + optionalAttrs + ; +in rec { + prepend = key: arr: job: { + ${key} = arr ++ (job.${key} or []); + }; + append = key: arr: job: { + ${key} = (job.${key} or []) ++ arr; + }; + prependToBeforeScript = prepend "before_script"; + appendToAfterScript = append "after_script"; + + # json is also valid yaml and this removes dependency on jq and/or remarshal + # (used in pkgs.formats.json and pkgs.formats.yaml respectively) + toYaml = name: value: + pipe value [ + builtins.toJSON + builtins.unsafeDiscardOutputDependency + builtins.unsafeDiscardStringContext + (builtins.toFile name) + ]; + toYamlPretty = (pkgs.formats.yaml {}).generate; + + customMapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); + + filterAttrsRec = pred: v: + if isAttrs v + then filterAttrs pred (mapAttrs (_path: filterAttrsRec pred) v) + else v; + + # filter job's variables to either only those containing store paths + # or those that do not + filterJobVariables = shouldContain: job: + concatMapAttrs ( + name: value: + optionalAttrs ((hasInfix builtins.storeDir value) == shouldContain) { + ${name} = value; + } + ) + (job.variables or {}); + + deepMerge = lhs: rhs: + lhs + // rhs + // (builtins.mapAttrs ( + rName: rValue: let + lValue = lhs.${rName} or null; + in + if builtins.isAttrs lValue && builtins.isAttrs rValue + then deepMerge lValue rValue + else if builtins.isList lValue && builtins.isList rValue + then lValue ++ rValue + else rValue + ) + rhs); + + unsetType = mkOptionType { + name = "unset"; + description = "unset"; + descriptionClass = "noun"; + check = _value: true; + }; + unset = { + _type = "unset"; + }; + isUnset = isType "unset"; + unsetOr = typ: + (types.either unsetType typ) + // { + inherit (typ) description getSubOptions; + }; + mkUnsetOption = opts: + mkOption (opts + // { + type = unsetOr opts.type; + default = opts.default or unset; + defaultText = literalExpression "unset"; + }); + eitherWithSubOptions = typ_one: typ_two: + (types.either typ_one typ_two) + // { + getSubOptions = + if (typ_one.getSubOptions "test" != {}) + then typ_one.getSubOptions + else typ_two.getSubOptions; + }; + + filterUnset = value: + if builtins.isAttrs value && !builtins.hasAttr "_type" value + then let + filteredAttrs = builtins.mapAttrs (_n: filterUnset) value; + in + filterAttrs (_name: value: (!isUnset value)) filteredAttrs + else if builtins.isList value + then builtins.filter (elem: !isUnset elem) (map filterUnset value) + else value; + + # args.pkgs so "pkgs" does not need to be passed all the time + stdenvMinimal = args.pkgs.stdenvNoCC.override { + cc = null; + preHook = ""; + allowedRequisites = null; + initialPath = with args.pkgs; [coreutils findutils]; + extraNativeBuildInputs = []; + }; +} diff --git a/lib/impl/jobDeps.nix b/lib/impl/jobDeps.nix new file mode 100644 index 0000000..302597d --- /dev/null +++ b/lib/impl/jobDeps.nix @@ -0,0 +1,35 @@ +{ + lib, + helpers, +}: let + inherit (lib) concatLines mapAttrsToList makeBinPath; + inherit (helpers) filterJobVariables stdenvMinimal; +in + { + key, + job, + nixConfig, + }: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = concatLines ( + mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + script = '' + export PATH="${makeBinPath (nixConfig.deps or [])}:$PATH"; + # variables containing nix derivations: + ${variableExports} + ''; + in + stdenvMinimal.mkDerivation { + name = "gitlab-ci-job-deps-${key}"; + dontUnpack = true; + installPhase = + # sh + '' + echo '${script}' > $out + chmod +x $out + ''; + passthru = { + inherit script; + }; + } diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix new file mode 100644 index 0000000..367a374 --- /dev/null +++ b/lib/impl/jobPatched.nix @@ -0,0 +1,42 @@ +{ + lib, + helpers, +}: let + inherit (lib) toList optionalAttrs optional; + inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables; +in + { + key, + job, + pipelineName, + nixConfig, + }: + if ! nixConfig.enable + then job + else + (builtins.removeAttrs job ["variables" "cache"]) + // (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job) + // (appendToAfterScript ["finalize_nix_ci"] job) + // (let + variables = + (filterJobVariables false job) + // optionalAttrs nixConfig.enableRunnerCache { + NIX_CI_CACHE_STRATEGY = "runner"; + }; + in + # filter empty variables + optionalAttrs (variables != {}) { + inherit variables; + }) + // (let + cache = + (toList (job.cache or [])) + ++ (optional nixConfig.enableRunnerCache { + key = nixConfig.runnerCacheKey; + paths = [".nix-cache/"]; + }); + in + # filter empty cache + optionalAttrs (cache != []) { + inherit cache; + }) diff --git a/lib/impl/jobRun.nix b/lib/impl/jobRun.nix new file mode 100644 index 0000000..4a1e0f6 --- /dev/null +++ b/lib/impl/jobRun.nix @@ -0,0 +1,47 @@ +{ + lib, + pkgs, + helpers, +}: let + inherit (lib) concatLines mapAttrsToList getExe; + inherit (helpers) filterJobVariables; +in + { + key, + job, + jobDeps, + }: let + variablesWithoutStorePaths = filterJobVariables false job; + variableExports = concatLines ( + mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths + ); + sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" (builtins.readFile ./sandbox_helper.sh); + actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" '' + # set up deps and environment variables containing store paths + . ${jobDeps} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + set -x + ${concatLines (job.before_script or [])} + { set +x; } 2>/dev/null + echo -e "\e[32mRunning script...\e[0m" + set -x + ${concatLines job.script} + { set +x; } 2>/dev/null + echo -e "\e[32mRunning after_script...\e[0m" + set -x + ${concatLines (job.after_script or [])} + { set +x; } 2>/dev/null + ''; + in + # this way the sandbox helper just needs to be built once + pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + exec ${getExe sandboxHelper} ${actualJobScript} $@ + '' + // { + passthru = { + inherit jobDeps actualJobScript; + }; + } diff --git a/lib/impl/modules/default.nix b/lib/impl/modules/default.nix new file mode 100644 index 0000000..9e33a7e --- /dev/null +++ b/lib/impl/modules/default.nix @@ -0,0 +1,32 @@ +{ + lib, + cilib, +}: rec { + inherit + (import ./root.nix { + inherit lib pipelineSubmodule soonixSubmodule; + }) + configSubmodule + nixCiSubmodule + ; + inherit + (import ./pipeline.nix { + inherit lib cilib jobSubmodule; + }) + pipelineConfigSubmodule + pipelineSubmodule + ; + inherit + (import ./job.nix { + inherit lib cilib; + }) + jobConfigSubmodule + jobSubmodule + ; + inherit + (import ./soonix.nix { + inherit lib cilib; + }) + soonixSubmodule + ; +} diff --git a/lib/impl/modules/job.nix b/lib/impl/modules/job.nix new file mode 100644 index 0000000..fd4fc62 --- /dev/null +++ b/lib/impl/modules/job.nix @@ -0,0 +1,663 @@ +{ + lib, + cilib, + ... +}: let + inherit (lib) mkOption types filterAttrs; + inherit (cilib.helpers) filterUnset mkUnsetOption eitherWithSubOptions; +in rec { + jobConfigSubmodule = {pipelineConfig, ...}: { + options = { + enable = mkOption { + description = '' + Transform this job to a nix-configured one. + ''; + type = types.bool; + default = pipelineConfig.nixJobsByDefault; + }; + deps = mkOption { + description = '' + Dependencies to inject into the job before running it. + ''; + type = types.listOf types.package; + default = []; + }; + enableRunnerCache = mkOption { + type = types.bool; + default = false; + description = '' + Cache this job using the GitLab Runner cache. + + !!! warning + useful for tiny jobs, but most of the time it just takes an eternity. + ''; + }; + runnerCacheKey = mkOption { + type = types.str; + default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; + description = '' + Cache key to use for the runner nix cache. Requires [`enableRunnerCache = true`](#pipelinesnamejobsnamenixenablerunnercache). + ''; + }; + }; + }; + + jobSubmodule = { + name, + config, + pipelineName, + pipelineConfig, + ... + }: let + # + # GITLAB OPTIONS + # + gitlabOptions = { + # see https://docs.gitlab.com/ci/yaml/ + after_script = mkUnsetOption { + type = types.listOf types.str; + description = '' + Override a set of commands that are executed after job. + + [Docs](https://docs.gitlab.com/ci/yaml/#after_script) + ''; + }; + allow_failure = mkUnsetOption { + type = eitherWithSubOptions types.bool (types.submodule { + options = { + exit_codes = mkUnsetOption { + type = types.either types.number (types.listOf types.number); + description = '' + Use `allow_failure.exit_codes` to control when a job should be allowed to fail. + The job is `allow_failure = true` for any of the listed exit codes, and `allow_failure = false` for any other exit code. + ''; + }; + }; + }); + description = '' + Allow job to fail. A failed job does not cause the pipeline to fail. + + [Docs](https://docs.gitlab.com/ci/yaml/#allow_failure) + ''; + }; + artifacts = mkUnsetOption { + # TODO: can be used in pipeline.default aswell + type = types.submodule { + options = { + paths = mkUnsetOption { + type = types.listOf types.str; + description = '' + Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can’t directly link outside it. + ''; + }; + excludes = mkUnsetOption { + type = types.listOf types.str; + description = '' + Use `exclude` to prevent files from being added to an artifacts archive. + ''; + }; + expire_in = mkUnsetOption { + type = types.str; + description = '' + Use `expire_in` to specify how long [job artifacts](https://docs.gitlab.com/ci/jobs/job_artifacts/) are stored before they expire and are deleted. + ''; + }; + expose_as = mkUnsetOption { + type = types.str; + description = '' + Use the `expose_as` keyword to [expose artifacts in the merge request UI](https://docs.gitlab.com/ci/jobs/job_artifacts/#link-to-job-artifacts-in-the-merge-request-ui). + ''; + }; + name = mkUnsetOption { + type = types.str; + description = '' + Use the `name` keyword to define the name of the created artifacts archive. You can specify a unique name for every archive. + ''; + }; + public = mkUnsetOption { + type = types.bool; + description = '' + Use `public` to determine whether the job artifacts should be publicly available. + ''; + }; + access = mkUnsetOption { + type = types.enum [ + "all" + "developer" + "maintainer" + "none" + ]; + description = '' + Use `access` to determine who can access the job artifacts from the GitLab UI or API. + This option does not prevent you from forwarding artifacts to downstream pipelines. + ''; + }; + reports = mkUnsetOption { + type = types.attrs; + description = '' + Use `reports` to collect artifacts generated by included templates in jobs. + ''; + }; + untracked = mkUnsetOption { + type = types.bool; + description = '' + Use `untracked` to add all Git untracked files as artifacts (along with the paths defined in `paths`). + `untracked` ignores configuration in the repository’s .gitignore, so matching artifacts in .gitignore are included. + ''; + }; + when = mkUnsetOption { + type = types.enum [ + "on_success" + "on_failure" + "always" + ]; + description = '' + Use `when` to upload artifacts on job failure or despite the failure. + ''; + }; + }; + }; + description = '' + List of files and directories to attach to a job on success. + + [Docs](https://docs.gitlab.com/ci/yaml/#artifacts) + ''; + }; + before_script = mkUnsetOption { + type = types.listOf types.str; + description = '' + Override a set of commands that are executed before job. + + [Docs](https://docs.gitlab.com/ci/yaml/#before_script) + ''; + }; + cache = mkUnsetOption { + # could be more granular + type = types.either (types.listOf types.attrs) types.attrs; + description = '' + List of files that should be cached between subsequent runs. + + [Docs](https://docs.gitlab.com/ci/yaml/#cache) + ''; + }; + coverage = mkUnsetOption { + type = types.str; + description = '' + Code coverage settings for a given job. + + [Docs](https://docs.gitlab.com/ci/yaml/#coverage) + ''; + }; + dast_configuration = mkUnsetOption { + type = types.attrs; + description = '' + Use configuration from DAST profiles on a job level. + + [Docs](https://docs.gitlab.com/ci/yaml/#dast_configuration) + ''; + }; + dependencies = mkUnsetOption { + type = types.listOf types.str; + description = '' + Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. + + [Docs](https://docs.gitlab.com/ci/yaml/#dependencies) + ''; + }; + environment = mkUnsetOption { + type = eitherWithSubOptions types.str (types.submodule { + options = { + name = mkUnsetOption { + type = types.str; + example = "production"; + description = '' + Set a name for an environment. + ''; + }; + url = mkUnsetOption { + type = types.str; + example = "https://prod.example.com"; + description = '' + Set a URL for an environment. + ''; + }; + on_stop = mkUnsetOption { + type = types.str; + example = "down"; + description = '' + Closing (stopping) environments can be achieved with the `on_stop` keyword defined under `environment`. + It declares a different job that runs to close the environment. + ''; + }; + action = mkUnsetOption { + type = types.enum ["start" "prepare" "stop" "verify" "access"]; + description = '' + Use the `action` keyword to specify how the job interacts with the environment. + ''; + }; + auto_stop_in = mkUnsetOption { + type = types.str; + description = '' + The `auto_stop_in` keyword specifies the lifetime of the environment. + When an environment expires, GitLab automatically stops it. + ''; + }; + kubernetes = mkUnsetOption { + type = types.attrs; # no we don't go that deep :D + description = '' + Use the `kubernetes` keyword to configure the + [dashboard for Kubernetes](https://docs.gitlab.com/ci/environments/kubernetes_dashboard/) and + [GitLab-managed Kubernetes resources](https://docs.gitlab.com/user/clusters/agent/managed_kubernetes_resources/) + for an environment. + ''; + }; + deployment_tier = mkUnsetOption { + type = types.enum [ + "production" + "staging" + "testing" + "development" + "other" + ]; + description = '' + Use the `deployment_tier` keyword to specify the tier of the deployment environment. + ''; + }; + }; + }); + description = '' + Name of an environment to which the job deploys. + See the implementation for nested options, or check out the docs: + + [Docs](https://docs.gitlab.com/ci/yaml/#environment) + ''; + example = { + name = "review/$CI_COMMIT_REF_SLUG"; + url = "https://$CI_COMMIT_REF_SLUG.review.example.com"; + action = "stop"; + auto_stop_in = "1 day"; + deployment_tier = "staging"; + }; + }; + extends = mkUnsetOption { + type = types.either types.str (types.listOf types.str); + description = '' + Configuration entries that this job inherits from. + + [Docs](https://docs.gitlab.com/ci/yaml/#extends) + ''; + }; + hooks = mkUnsetOption { + type = types.attrs; + description = '' + Use `hooks` to specify lists of commands to execute on the runner at certain stages of job execution, + like before retrieving the Git repository. + + [Docs](https://docs.gitlab.com/ci/yaml/#hooks) + ''; + }; + identity = mkUnsetOption { + type = types.str; + description = '' + Authenticate with third party services using identity federation. + + [Docs](https://docs.gitlab.com/ci/yaml/#identity) + ''; + }; + id_tokens = mkUnsetOption { + type = types.attrs; + description = '' + Use `id_tokens` to create [ID tokens](https://docs.gitlab.com/ci/secrets/id_token_authentication/) to authenticate with third party services + + [Docs](https://docs.gitlab.com/ci/yaml/#id_tokens) + ''; + example = { + ID_TOKEN_1.aud = "https://vault.example.com"; + ID_TOKEN_2.aud = [ + "https://gcp.com" + "https://aws.com" + ]; + SIGSTORE_ID_TOKEN.aud = "sigstore"; + }; + }; + image = mkOption { + # could be more granular + type = types.either types.str types.attrs; + description = '' + Container/OCI image to use for this job. + + !!! warning + Setting this will mess with Nix-GitLab-CI, so be careful and only use for non-nix jobs. + ''; + default = "$NIX_CI_IMAGE"; + example = { + name = "super/sql:experimental"; + entrypoint = [""]; + pull_policy = "if-not-present"; + docker = { + platform = "arm64/v8"; + user = "dave"; + }; + kubernetes.user = "1001"; + }; + }; + "inherit" = mkUnsetOption { + type = types.submodule { + options = { + default = mkUnsetOption { + type = types.either types.bool (types.listOf types.str); + description = '' + Use `inherit.default` to control the inheritance of [default keywords](https://docs.gitlab.com/ci/yaml/#default). + ''; + }; + variables = mkUnsetOption { + type = types.either types.bool (types.listOf types.str); + description = '' + Use `inherit.variables` to control the inheritance of [default variables](https://docs.gitlab.com/ci/yaml/#default-variables). + ''; + }; + }; + }; + description = '' + Select which global defaults all jobs inherit. + + [Docs](https://docs.gitlab.com/ci/yaml/#inherit) + ''; + }; + interruptible = mkUnsetOption { + type = types.bool; + description = '' + Defines if a job can be canceled when made redundant by a newer run. + + [Docs](https://docs.gitlab.com/ci/yaml/#interruptible) + ''; + }; + manual_confirmation = mkUnsetOption { + type = types.str; + description = '' + Define a custom confirmation message for a manual job. + + [Docs](https://docs.gitlab.com/ci/yaml/#manual_confirmation) + ''; + }; + needs = mkUnsetOption { + # could be done more granular + type = types.listOf (types.either types.str types.attrs); + description = '' + Execute jobs earlier than the stage ordering. + + [Docs](https://docs.gitlab.com/ci/yaml/#needs) + ''; + }; + pages = mkUnsetOption { + type = eitherWithSubOptions types.bool (types.submodule { + options = { + publish = mkUnsetOption { + type = types.str; + description = '' + Use `pages.publish` to configure the content directory of a [`pages` job](https://docs.gitlab.com/ci/yaml/#pages). + ''; + }; + path_prefix = mkUnsetOption { + type = types.str; + description = '' + Use `pages.path_prefix` to configure a path prefix for [parallel deployments](https://docs.gitlab.com/user/project/pages/#parallel-deployments) of GitLab Pages. + ''; + }; + expire_in = mkUnsetOption { + type = types.str; + description = '' + Use `expire_in` to specify how long a deployment should be available before it expires. + After the deployment is expired, it’s deactivated by a cron job running every 10 minutes. + ''; + }; + }; + }); + description = '' + Upload the result of a job to use with GitLab Pages. + + [Docs](https://docs.gitlab.com/ci/yaml/#pages) + ''; + }; + parallel = mkUnsetOption { + type = eitherWithSubOptions types.number (types.listOf (types.submodule { + options = { + matrix = mkUnsetOption { + type = types.attrs; + description = '' + Use `parallel.matrix` to run a job multiple times in parallel in a single pipeline, but with different variable values for each instance of the job. + ''; + }; + }; + })); + description = '' + How many instances of a job should be run in parallel. + + [Docs](https://docs.gitlab.com/ci/yaml/#parallel) + ''; + example = { + matrix = [ + { + PROVIDER = "aws"; + STACK = [ + "monitoring" + "app1" + "app2" + ]; + } + { + PROVIDER = "ovh"; + STACK = ["monitoring" "backup" "app"]; + } + { + PROVIDER = ["gcp" "vultr"]; + STACK = ["data" "processing"]; + } + ]; + }; + }; + release = mkUnsetOption { + # could be more granular + type = types.attrs; + description = '' + Instructs the runner to generate a [release](https://docs.gitlab.com/user/project/releases/) object. + + [Docs](https://docs.gitlab.com/ci/yaml/#release) + ''; + }; + resource_group = mkUnsetOption { + type = types.str; + description = '' + Limit job concurrency. + + [Docs](https://docs.gitlab.com/ci/yaml/#resource_group) + ''; + }; + retry = mkUnsetOption { + type = eitherWithSubOptions (types.ints.between 0 2) (types.submodule { + options = { + exit_codes = mkUnsetOption { + type = types.either types.int (types.listOf types.int); + description = '' + Use `retry.exit_codes` with `retry.max` to retry jobs for only specific failure cases. + ''; + }; + when = mkUnsetOption { + type = types.either types.str (types.listOf types.str); + description = '' + Use `retry.when` with `retry.max` to retry jobs for only specific failure cases. + ''; + }; + max = mkUnsetOption { + type = types.ints.between 0 2; + description = '' + `retry.max` is the maximum number of retries, like retry, and can be 0, 1, or 2. + ''; + }; + }; + }); + description = '' + When and how many times a job can be auto-retried in case of a failure. + + [Docs](https://docs.gitlab.com/ci/yaml/#retry) + ''; + }; + rules = mkUnsetOption { + # could be more granular + type = types.listOf types.attrs; + description = '' + List of conditions to evaluate and determine selected attributes of a job, and whether or not it’s created. + + [Docs](https://docs.gitlab.com/ci/yaml/#rules) + ''; + }; + script = mkOption { + type = types.listOf types.str; + description = '' + Shell script that is executed by a runner. + + [Docs](https://docs.gitlab.com/ci/yaml/#script) + ''; + }; + secrets = mkUnsetOption { + # could be more granular + type = types.attrs; + description = '' + The CI/CD secrets the job needs. + + [Docs](https://docs.gitlab.com/ci/yaml/#secrets) + ''; + }; + services = mkUnsetOption { + # could be more granular + type = types.attrs; + description = '' + Use Docker services images. + + [Docs](https://docs.gitlab.com/ci/yaml/#services) + ''; + }; + stage = mkOption { + type = types.str; + description = '' + Defines a job stage. + + [Docs](https://docs.gitlab.com/ci/yaml/#stage) + ''; + }; + tags = mkUnsetOption { + type = types.listOf types.str; + description = '' + List of tags that are used to select a runner. + + [Docs](https://docs.gitlab.com/ci/yaml/#tags) + ''; + }; + timeout = mkUnsetOption { + type = types.str; + description = '' + Define a custom job-level timeout that takes precedence over the project-wide setting. + + [Docs](https://docs.gitlab.com/ci/yaml/#timeout) + ''; + }; + trigger = mkUnsetOption { + # could be more granular + type = types.either types.str types.attrs; + description = '' + Defines a downstream pipeline trigger. + + [Docs](https://docs.gitlab.com/ci/yaml/#trigger) + ''; + }; + when = mkUnsetOption { + type = types.enum ["on_success" "on_failure" "never" "always" "manual" "delayed"]; + description = '' + When to run job. See also [`manual_confirmation`](#pipelinesnamejobsnamemanual_confirmation) + + [Docs](https://docs.gitlab.com/ci/yaml/#when) + ''; + }; + variables = mkUnsetOption { + type = types.attrsOf types.str; + description = '' + You can use job variables in commands in the job’s `script`, `before_script`, or `after_script` sections, and also with some job keywords. + Check the **Supported values** section of each job keyword to see if it supports variables. + + [Docs](https://docs.gitlab.com/ci/yaml/#job-variables) + ''; + }; + }; + in { + options = + { + nix = mkOption { + description = '' + Nix-GitLab-CI config options for this job. + ''; + type = types.submoduleWith { + modules = [jobConfigSubmodule]; + specialArgs.pipelineConfig = pipelineConfig; + }; + default = {}; + }; + finalConfig = mkOption { + description = '' + Final configuration of this job. (readonly) + ''; + readOnly = true; + internal = true; + type = types.attrs; + }; + depsDrv = mkOption { + description = '' + Derivation containing all the dependencies of this job. (readonly) + ''; + readOnly = true; + internal = true; + type = types.package; + }; + runnerDrv = mkOption { + description = '' + Derivation containing the script for running the job locally. (readonly) + ''; + readOnly = true; + internal = true; + type = types.package; + }; + packages = mkOption { + description = '' + Final packages for this job, eg. for running the job or getting it's deps. (readonly) + ''; + readOnly = true; + internal = true; + type = types.attrsOf types.package; + }; + } + // gitlabOptions; + config = let + attrsToKeep = builtins.attrNames gitlabOptions; + job = filterUnset (filterAttrs (n: _v: builtins.elem n attrsToKeep) config); + in { + finalConfig = cilib.mkJobPatched { + key = name; + nixConfig = config.nix; + inherit job pipelineName; + }; + depsDrv = cilib.mkJobDeps { + key = name; + nixConfig = config.nix; + inherit job; + }; + runnerDrv = cilib.mkJobRun { + key = name; + jobDeps = config.depsDrv; + inherit job; + }; + packages = { + "gitlab-ci:pipeline:${pipelineName}:job-deps:${name}" = config.depsDrv; + "gitlab-ci:pipeline:${pipelineName}:job:${name}" = config.runnerDrv; + }; + }; + }; +} diff --git a/lib/impl/modules/pipeline.nix b/lib/impl/modules/pipeline.nix new file mode 100644 index 0000000..32b9e0c --- /dev/null +++ b/lib/impl/modules/pipeline.nix @@ -0,0 +1,228 @@ +{ + lib, + cilib, + jobSubmodule, + ... +}: let + inherit (lib) mkOption types filterAttrs mergeAttrsList mapAttrs mkRenamedOptionModule literalExpression; + inherit (cilib.helpers) filterUnset unset mkUnsetOption toYaml toYamlPretty eitherWithSubOptions; + + pipelineConfigSubmodule = {rootConfig, ...}: { + options = { + nixJobsByDefault = mkOption { + description = '' + Whether to transform all jobs to nix-configured jobs by default. + If false, you need to set `nix.enable` for each job you want to be transformed. + ''; + type = types.bool; + default = rootConfig.nixJobsByDefault; + }; + }; + }; + + pipelineSubmodule = { + name, + config, + rootConfig, + ... + }: let + # + # GITLAB OPTIONS + # + gitlabOptions = { + default = mkOption { + type = types.submodule { + # allow other keys aswell, maybe define them in the future? https://docs.gitlab.com/ci/yaml/#default + freeformType = types.anything; + options = { + image = mkUnsetOption { + type = types.str; + description = '' + Default image to use for this entire pipeline. + + !!! note + Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). + ''; + }; + services = mkUnsetOption { + type = types.listOf types.anything; + description = '' + !!! note + Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). + ''; + }; + cache = mkUnsetOption { + # could be more granular + type = types.either (types.listOf types.attrs) types.attrs; + description = '' + !!! note + Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). + ''; + }; + before_script = mkUnsetOption { + type = types.listOf types.str; + description = '' + !!! note + Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). + ''; + }; + after_script = mkUnsetOption { + type = types.listOf types.str; + description = '' + !!! note + Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). + ''; + }; + }; + }; + # required for it to show up in the docs, but not in the config + default = unset; + defaultText = literalExpression "unset"; + description = '' + Custom default values for job keywords. + ''; + }; + include = mkUnsetOption { + type = types.attrs; + description = '' + Import configuration from other YAML files. + + [Docs](https://docs.gitlab.com/ci/yaml/#include) + ''; + }; + stages = mkOption { + type = types.listOf types.str; + default = []; + # .pre and .post always exist + apply = val: [".pre"] ++ val ++ [".post"]; + description = '' + The names and order of the pipeline stages. + + [Docs](https://docs.gitlab.com/ci/yaml/#stages) + + !!! note + `.pre` and `.post` are added automatically. + ''; + }; + variables = mkUnsetOption { + # default/global variables can have descriptions etc. + type = types.attrsOf (eitherWithSubOptions types.str (types.submodule { + options = { + description = mkUnsetOption { + type = types.str; + description = '' + Use the `description` keyword to define a description for a default variable. + ''; + }; + value = mkUnsetOption { + type = types.str; + description = '' + Use the `value` keyword to define a pipeline-level (default) variable’s value. + ''; + }; + options = mkUnsetOption { + type = types.listOf types.str; + description = '' + Use `options` to define an array of values that are [selectable in the UI when running a pipeline manually](https://docs.gitlab.com/ci/pipelines/#configure-a-list-of-selectable-prefilled-variable-values). + ''; + }; + expand = mkUnsetOption { + type = types.bool; + description = '' + Use the `expand` keyword to configure a variable to be expandable or not. + ''; + }; + }; + })); + description = '' + Define default CI/CD variables for all jobs in the pipeline. + Supports strings or attrs as values, for more info see [here](https://docs.gitlab.com/ci/yaml/#variablesdescription). + + [Docs](https://docs.gitlab.com/ci/yaml/#default-variables) + ''; + }; + workflow = mkUnsetOption { + type = types.attrs; + description = '' + Control what types of pipeline run. + + [Docs](https://docs.gitlab.com/ci/yaml/#workflow) + ''; + }; + }; + in { + _file = ./pipeline.nix; + # from https://docs.gitlab.com/ci/yaml/#default + imports = builtins.map (val: mkRenamedOptionModule [val] ["default" val]) [ + "after_script" + "artifacts" + "before_script" + "cache" + "hooks" + "id_tokens" + "image" + "interruptible" + "retry" + "services" + "tags" + "timeout" + ]; + options = + { + nix = mkOption { + description = '' + Nix-GitLab-CI config options for this pipeline. + ''; + type = types.submoduleWith { + modules = [pipelineConfigSubmodule]; + specialArgs.rootConfig = rootConfig; + }; + default = {}; + }; + finalConfig = mkOption { + description = '' + Final config of the pipeline. (readonly) + ''; + readOnly = true; + type = types.attrs; + }; + packages = mkOption { + description = '' + Final packages for use in CI. (readonly) + ''; + readOnly = true; + type = types.attrsOf types.package; + }; + # jobs are nested to make distinguishing them from other keys in the ci config easier + jobs = mkOption { + description = '' + Jobs for this pipeline. + ''; + type = types.attrsOf (types.submoduleWith { + modules = [jobSubmodule]; + specialArgs = { + pipelineName = name; + pipelineConfig = config.nix; + }; + }); + default = {}; + }; + } + // gitlabOptions; + config = let + attrsToKeep = builtins.attrNames gitlabOptions; + in { + finalConfig = + (filterUnset (filterAttrs (n: _v: builtins.elem n attrsToKeep) config)) + // mapAttrs (_name: value: value.finalConfig) config.jobs; + packages = + { + "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-config.json" config.finalConfig; + "gitlab-ci:pipeline:${name}:pretty" = toYamlPretty "gitlab-ci-config.yml" config.finalConfig; + } + // mergeAttrsList (map (job: job.packages) (builtins.attrValues config.jobs)); + }; + }; +in { + inherit pipelineSubmodule pipelineConfigSubmodule; +} diff --git a/lib/impl/modules/root.nix b/lib/impl/modules/root.nix new file mode 100644 index 0000000..bd99050 --- /dev/null +++ b/lib/impl/modules/root.nix @@ -0,0 +1,74 @@ +{ + lib, + soonixSubmodule, + pipelineSubmodule, + ... +}: let + inherit (lib) mkOption types foldr; +in rec { + configSubmodule = { + options = { + soonix = mkOption { + description = '' + Configure the soonix `.gitlab-ci.yml` generation. + ''; + type = types.submodule soonixSubmodule; + default = {}; + }; + nixJobsByDefault = mkOption { + description = '' + Whether to transform all jobs to nix-configured jobs by default. + If false, you need to set [`nix.enable`](#pipelinesnamejobsnamenixenable) + for each job you want to be transformed. + ''; + type = types.bool; + default = true; + }; + }; + }; + + nixCiSubmodule = {config, ...}: { + options = { + config = mkOption { + description = '' + Configuration of Nix-GitLab-CI itself. + ''; + type = types.submodule configSubmodule; + default = {}; + }; + pipelines = mkOption { + description = '' + Defines all pipelines. + ''; + type = types.attrsOf (types.submoduleWith { + modules = [pipelineSubmodule]; + specialArgs.rootConfig = config.config; + }); + default = {}; + }; + + packages = mkOption { + description = '' + Final packages for use in CI. (readonly) + ''; + readOnly = true; + type = types.attrsOf types.package; + }; + soonix = mkOption { + description = '' + Soonix config for generating `.gitlab-ci.yml`. (readonly) + + See [`config.soonix`](#configsoonix) for configuring this. + ''; + readOnly = true; + type = types.attrs; + }; + }; + config = { + packages = foldr (pipeline: acc: acc // pipeline) {} ( + map (pipeline: pipeline.packages) (builtins.attrValues config.pipelines) + ); + soonix = config.config.soonix.finalConfig; + }; + }; +} diff --git a/lib/impl/modules/soonix.nix b/lib/impl/modules/soonix.nix new file mode 100644 index 0000000..d23d727 --- /dev/null +++ b/lib/impl/modules/soonix.nix @@ -0,0 +1,69 @@ +{ + lib, + cilib, + ... +}: let + inherit (lib) mkOption types; +in { + soonixSubmodule = {config, ...}: { + options = { + componentVersion = mkOption { + description = '' + CI/CD component version. Also get's passed to inputs → version. + ''; + type = types.str; + default = cilib.version; + }; + componentUrl = mkOption { + description = '' + CI/CD component url. + ''; + type = types.str; + default = "gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci"; + }; + componentInputs = mkOption { + description = '' + Extra inputs to pass to the CI/CD component. + ''; + type = types.attrs; + default = {}; + }; + extraData = mkOption { + description = '' + Extra data to include in the `.gitlab-ci.yml` file. + ''; + type = types.attrs; + default = {}; + }; + + finalConfig = mkOption { + internal = true; + type = types.attrs; + }; + }; + config.finalConfig = { + opts.format = "yaml"; + hook = { + mode = "copy"; + gitignore = false; + }; + output = ".gitlab-ci.yml"; + generator = "nix"; + data = + cilib.helpers.deepMerge + { + include = [ + { + component = "${config.componentUrl}@${config.componentVersion}"; + inputs = + { + version = config.componentVersion; + } + // config.componentInputs; + } + ]; + } + config.extraData; + }; + }; +} diff --git a/lib/impl/pipeline.nix b/lib/impl/pipeline.nix new file mode 100644 index 0000000..4d9b185 --- /dev/null +++ b/lib/impl/pipeline.nix @@ -0,0 +1,61 @@ +{ + lib, + helpers, + mkJobDeps, + mkJobRun, + mkJobPatched, +}: let + inherit (lib) assertMsg; + inherit (helpers) filterAttrsRec customMapAttrs toYaml toYamlPretty; +in + { + name, + pipeline, + nixConfig, + }: let + jobs = filterAttrsRec (_n: v: v != null) pipeline.jobs; + rest = filterAttrsRec (_n: v: v != null) (builtins.removeAttrs pipeline ["jobs"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + customMapAttrs (key: job: { + name = "gitlab-ci:pipeline:${name}:job-deps:${key}"; + value = mkJobDeps {inherit key job nixConfig;}; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + customMapAttrs (key: job: { + name = "gitlab-ci:pipeline:${name}:job:${key}"; + value = mkJobRun { + inherit key job nixConfig; + jobDeps = jobsMappedForDeps."gitlab-ci:pipeline:${name}:job-deps:${key}"; + }; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + customMapAttrs (key: job: { + name = key; + value = assert assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + mkJobPatched { + inherit key job nixConfig; + pipelineName = name; + }; + }) + jobs; + in { + packages = + # gitlab-ci:pipeline: + # gitlab-ci:pipeline::job: + # gitlab-ci:pipeline::job-deps: + { + "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-${name}.yml" (rest // jobsPatched); + "gitlab-ci:pipeline:${name}:pretty" = toYamlPretty "gitlab-ci-${name}.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript; + finalConfig = rest // jobsPatched; + } diff --git a/lib/impl/sandbox_helper.sh b/lib/impl/sandbox_helper.sh new file mode 100644 index 0000000..8b76ae1 --- /dev/null +++ b/lib/impl/sandbox_helper.sh @@ -0,0 +1,75 @@ +echo -e "\e[32mSetting up...\e[0m" + +actualJobScript=$1 +shift + +INCLUDE_DIRTY=false +NO_SANDBOX=false +KEEP_TMP=false +KEEP_ENV="" +# parse flags +while [[ $# -gt 0 ]]; do + case "$1" in + --include-dirty) + INCLUDE_DIRTY=true + shift + ;; + --no-sandbox) + NO_SANDBOX=true + shift + ;; + --keep-tmp) + KEEP_TMP=true + shift + ;; + --keep-env) + KEEP_ENV="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + echo "use --include-dirty, --no-sandbox, --keep-tmp and --keep-env " >&2 + exit 1 + ;; + esac +done + +if [ $NO_SANDBOX = false ]; then + echo "Running with simple sandboxing" + NGCI_TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX") + if [ $KEEP_TMP = false ]; then + trap "rm -rf '$NGCI_TMPDIR'" EXIT + else + echo "Temp dir will be preserved at: $NGCI_TMPDIR" + fi + + # check if dirty + DIRTY_PATCH="" + if ! git diff --quiet && ! git diff --staged --quiet; then + echo "Warning: working tree is dirty." + DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch") + git diff --staged > "$DIRTY_PATCH" + trap "rm -f '$DIRTY_PATCH'" EXIT + fi + git clone . $NGCI_TMPDIR + pushd $NGCI_TMPDIR >/dev/null + if [[ ! -z "$DIRTY_PATCH" && $INCLUDE_DIRTY = true ]]; then + echo "Copying dirty changes..." + git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes" + git add . # required so the files are staged again + fi + + echo "Running job in $NGCI_TMPDIR" + env -i $( + if [[ -n "$KEEP_ENV" ]]; then + IFS=',' read -ra VARS <<< "$KEEP_ENV" + for var in "${VARS[@]}"; do + printf '%s=%q ' "$var" "${!var}" + done + fi + ) bash $actualJobScript + popd >/dev/null +else + exec $actualJobScript +fi + diff --git a/lib/utils.nix b/lib/utils.nix index a40dadd..ae15284 100644 --- a/lib/utils.nix +++ b/lib/utils.nix @@ -1,46 +1,46 @@ -{ - mkUtils = {pkgs, ...}: { - commitAndPushFiles = { - message, - files ? [], - }: jobArgs: - jobArgs - // { - before_script = - (jobArgs.before_script or []) - ++ [ - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" - eval "$(ssh-agent -s)" >/dev/null; - mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; - ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; - echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; - git config --global user.email "$GIT_EMAIL" >/dev/null; - git config --global user.name "$GIT_NAME" >/dev/null; - export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; - git remote rm origin && git remote add origin ''${CI_PUSH_REPO} - echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" - '' - ]; - script = let - addScript = - if builtins.length files == 0 - then "" - else "git add ${builtins.concatStringsSep " " files}"; - in - (jobArgs.script or []) - ++ [ - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" - ${addScript} - git diff --cached --exit-code >/dev/null && - echo "Nothing to commit" || - git commit -m "${message}" --no-verify; - git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip - echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" - '' - ]; - nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; - }; - }; +{pkgs, ...}: { + commitAndPushFiles = { + message, + files ? [], + }: jobArgs: + jobArgs + // { + before_script = + (jobArgs.before_script or []) + ++ [ + # sh + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" + eval "$(ssh-agent -s)" >/dev/null; + mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; + ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; + echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; + git config --global user.email "$GIT_EMAIL" >/dev/null; + git config --global user.name "$GIT_NAME" >/dev/null; + export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; + git remote rm origin && git remote add origin ''${CI_PUSH_REPO} + echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" + '' + ]; + script = let + addScript = + if builtins.length files == 0 + then "" + else "git add ${builtins.concatStringsSep " " files}"; + in + (jobArgs.script or []) + ++ [ + # sh + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" + ${addScript} + git diff --cached --exit-code >/dev/null && + echo "Nothing to commit" || + git commit -m "${message}" --no-verify; + git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip + echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" + '' + ]; + nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; + }; } diff --git a/nix/packages/pkgs.nix b/nix/packages/pkgs.nix new file mode 100644 index 0000000..34d07aa --- /dev/null +++ b/nix/packages/pkgs.nix @@ -0,0 +1,48 @@ +{ + inputs, + system, + ... +}: let + inherit (inputs) pkgs; +in rec { + setupScript = pkgs.writeShellScriptBin "setup_nix_ci" (builtins.readFile ./scripts/setup_nix_ci.sh); + finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" (builtins.readFile ./scripts/finalize_nix_ci.sh); + image = pkgs.dockerTools.buildImage { + name = "nix-ci"; + fromImage = let + hashes = { + "x86_64-linux" = "sha256-kJ7dqje5o1KPr3RDZ7/THbhMSoiCU1C/7HshDrNfwnM="; + "aarch64-linux" = "sha256-jz+Z3Ji+hy5d9ImOh/YOKCqy9P9/cseSov+5J/O95bg="; + }; + # check digest of tags like nixos-24.11-aarch64-linux etc. + digests = { + "x86_64-linux" = "sha256:345f210dea4cbd049e2d01d13159c829066dfb6e273cdd49ea878186d17b19f7"; + "aarch64-linux" = "sha256:66163fdf446d851416dd4e9be28c0794d9c2550214a57a846957699a3f5747f6"; + }; + hash = hashes.${system} or (throw "Unsupported system"); + imageDigest = digests.${system} or (throw "Unsupported system"); + in + pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-flakes"; + inherit hash imageDigest; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = with pkgs; + [ + gitMinimal + gnugrep + gnused + coreutils + diffutils + cachix + attic-client + ] + ++ [ + setupScript + finalizeScript + ]; + pathsToLink = ["/bin"]; + }; + }; +} diff --git a/nix/packages/scripts/finalize_nix_ci.sh b/nix/packages/scripts/finalize_nix_ci.sh new file mode 100644 index 0000000..ecc2211 --- /dev/null +++ b/nix/packages/scripts/finalize_nix_ci.sh @@ -0,0 +1,40 @@ +echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci[collapsed=true]\\r\\e[0KFinalizing Nix CI..." + nix path-info --all > /tmp/nix-store-after + echo "Finding new paths..." + NEW_PATHS=$(diff --new-line-format="%L" \ + --old-line-format="" --unchanged-line-format="" \ + /tmp/nix-store-before /tmp/nix-store-after) + COUNT=$(wc -l <<<"$NEW_PATHS") + + if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then + export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}"; + fi + + if [ -z "$NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($NIX_CI_CACHE_STRATEGY)" + echo -n "$NEW_PATHS" | { + case "$NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + # add ^* to all store paths ending in .drv (prevent warning log spam) + sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true + ;; + "attic") + attic push --stdin ci:$ATTIC_CACHE || true + ;; + "cachix") + cachix push $CACHIX_CACHE || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'" + ;; + esac + } + echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" + else + echo "Caching disabled, not uploading $COUNT new store entries..." + fi +echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K" diff --git a/nix/packages/scripts/setup_nix_ci.sh b/nix/packages/scripts/setup_nix_ci.sh new file mode 100644 index 0000000..eaf189b --- /dev/null +++ b/nix/packages/scripts/setup_nix_ci.sh @@ -0,0 +1,48 @@ +echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" + nix path-info --all > /tmp/nix-store-before + + if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then + export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}"; + echo "NIX_CI_CACHE_STRATEGY was set to auto, selected '$NIX_CI_CACHE_STRATEGY' for this job" + fi + + if [ -z "$NIX_CI_DISABLE_CACHE" ]; then + echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($NIX_CI_CACHE_STRATEGY)" + case "$NIX_CI_CACHE_STRATEGY" in + "runner") + export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} + echo "Runner Cache: $RUNNER_CACHE" + export NIX_CONFIG="$NIX_CONFIG + extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true + extra-substituters = $RUNNER_CACHE?priority=10&trusted=true + " + ;; + "attic") + echo "Attic Cache: $ATTIC_CACHE" + attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true + attic use "$ATTIC_CACHE" || true + ;; + "cachix") + echo "Cachix Cache: $CACHIX_CACHE" + cachix use "$CACHIX_CACHE" || true + ;; + "none") + echo "Cache strategy is none, doing nothing..." + ;; + *) + echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'" + ;; + esac + echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K" + else + echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..." + fi + + # load the job's deps only if the name was passed + if [[ ! -z $1 ]]; then + echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job" + nix build .#$1 + source $(readlink -f result) + echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" + fi +echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" diff --git a/nix/repo/ci.nix b/nix/repo/ci.nix new file mode 100644 index 0000000..157ede7 --- /dev/null +++ b/nix/repo/ci.nix @@ -0,0 +1,100 @@ +{inputs, ...}: let + inherit (inputs) cilib; +in + cilib.mkCI { + config.soonix = { + componentUrl = "$CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci"; + componentVersion = "$CI_COMMIT_SHORT_SHA"; + componentInputs.cache_files = ["flake.*" "nix/repo/ci.nix"]; + # bootstrapping still needs to be done in the gitlab-ci.yml directly, + # the child pipeline can then use the built images to test them + extraData = { + stages = ["build-images" "build" "trigger"]; + variables.NIX_CI_IMAGE = "$CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA"; + "build:image" = { + stage = "build-images"; + parallel.matrix = [ + {ARCH = ["x86_64-linux" "aarch64-linux"];} + ]; + image = "nixpkgs/nix-flakes:latest"; + script = ["nix build .#image --system $ARCH"]; + after_script = ["install -D result dist/nix-ci-$ARCH.tar.gz"]; + artifacts.paths = ["dist"]; + }; + "deploy:image" = { + stage = "build-images"; + image = "nixpkgs/nix-flakes:latest"; + needs = ["build:image"]; + before_script = [ + # sh + '' + nix profile install nixpkgs#buildah + export PATH="$PATH:$HOME/.nix-profile/bin" + export REGISTRY_AUTH_FILE=''${HOME}/auth.json + echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + mkdir -p /etc/containers && echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json + mkdir -p /var/tmp + '' + ]; + script = [ + # sh + '' + export NORMALIZED_BRANCH=''${CI_COMMIT_BRANCH/\//-} + buildah manifest create localhost/nix-ci + buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz + buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:''${CI_COMMIT_SHORT_SHA} + # branches + if [ -z "$CI_COMMIT_TAG" ]; then + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:''${NORMALIZED_BRANCH/main/latest} + fi + # tags + if [ -n "$CI_COMMIT_TAG" ]; then + buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:''${CI_COMMIT_TAG} + fi + '' + ]; + }; + }; + }; + pipelines."default" = { + stages = ["test" "build" "deploy"]; + jobs = { + "test" = { + stage = "test"; + script = [ + "nix run .#tests -- --junit=junit.xml" + ]; + allow_failure = true; + artifacts = { + when = "always"; + reports.junit = "junit.xml"; + }; + }; + "docs" = { + stage = "build"; + script = [ + # sh + '' + nix build .#docs:default + mkdir -p public + cp -r result/. public/ + '' + ]; + artifacts.paths = ["public"]; + }; + "pages" = { + nix.enable = false; + image = "alpine:latest"; + stage = "deploy"; + script = ["true"]; + artifacts.paths = ["public"]; + rules = [ + { + "if" = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"; + } + ]; + }; + }; + }; + } diff --git a/nix/repo/devShells.nix b/nix/repo/devShells.nix new file mode 100644 index 0000000..10889f9 --- /dev/null +++ b/nix/repo/devShells.nix @@ -0,0 +1,35 @@ +{ + cell, + inputs, + ... +}: let + inherit (inputs) pkgs devshell treefmt soonix; + inherit (cell) ci; +in { + default = devshell.mkShell { + imports = [soonix.devshellModule]; + packages = [ + pkgs.nil + (treefmt.mkWrapper pkgs { + programs = { + alejandra.enable = true; + deadnix.enable = true; + statix.enable = true; + mdformat.enable = true; + yamlfmt.enable = true; + }; + settings.formatter = { + yamlfmt.excludes = ["templates/nix-gitlab-ci.yml" ".gitlab-ci.yml"]; + mdformat.command = let + pkg = pkgs.python3.withPackages (p: [ + p.mdformat + p.mdformat-mkdocs + ]); + in "${pkg}/bin/mdformat"; + }; + }) + ]; + + soonix.hooks."ci" = ci.soonix; + }; +} diff --git a/nix/repo/docs.nix b/nix/repo/docs.nix new file mode 100644 index 0000000..052b177 --- /dev/null +++ b/nix/repo/docs.nix @@ -0,0 +1,80 @@ +{inputs, ...}: let + inherit (inputs) pkgs cilib doclib; + + optionsDoc = doclib.mkOptionDocs { + module = cilib.modules.nixCiSubmodule; + roots = [ + { + url = "https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/blob/main/lib"; + path = "${inputs.self}/lib"; + } + ]; + }; + optionsDocs = pkgs.runCommand "options-docs" {} '' + mkdir -p $out + ln -s ${optionsDoc} $out/options.md + ''; +in + (doclib.mkDocs { + docs."default" = { + base = "${inputs.self}"; + path = "${inputs.self}/docs"; + material = { + enable = true; + colors = { + primary = "deep orange"; + accent = "orange"; + }; + umami = { + enable = true; + src = "https://analytics.tf/umami"; + siteId = "28f7c904-db22-4c2b-9ee4-ed42e14b6db9"; + domains = ["nix-gitlab-ci.projects.tf"]; + }; + }; + macros = { + enable = true; + includeDir = toString optionsDocs; + }; + config = { + site_name = "Nix-GitLab-CI"; + site_url = "https://nix-gitlab-ci.projects.tf"; + repo_name = "TECHNOFAB/nix-gitlab-ci"; + repo_url = "https://gitlab.com/TECHNOFAB/nix-gitlab-ci"; + extra_css = ["style.css"]; + theme = { + logo = "images/logo.svg"; + icon.repo = "simple/gitlab"; + favicon = "images/logo.svg"; + }; + nav = [ + {"Introduction" = "index.md";} + {"Setup" = "setup.md";} + {"Usage" = "usage.md";} + {"CI/CD Component" = "cicd_component.md";} + {"Environment Variables" = "environment_variables.md";} + {"Caching" = "caching.md";} + {"Multiple Pipelines" = "multi_pipeline.md";} + {"Soonix Integration" = "soonix.md";} + {"Utilities" = "utilities.md";} + {"Kubernetes Runner Example" = "kubernetes_runner.md";} + {"Example Configs" = "examples.md";} + {"Options" = "options.md";} + ]; + markdown_extensions = [ + { + "pymdownx.highlight".pygments_lang_class = true; + } + "pymdownx.inlinehilite" + "pymdownx.snippets" + "pymdownx.superfences" + "pymdownx.escapeall" + "fenced_code" + "admonition" + ]; + }; + }; + }).packages + // { + inherit optionsDocs; + } diff --git a/nix/repo/flake.lock b/nix/repo/flake.lock new file mode 100644 index 0000000..e5029e0 --- /dev/null +++ b/nix/repo/flake.lock @@ -0,0 +1,99 @@ +{ + "nodes": { + "devshell-lib": { + "locked": { + "dir": "lib", + "lastModified": 1758204313, + "narHash": "sha256-ainbY0Oajb1HMdvy+A8QxF/P5qwcbEzJGEY5pzKdDdc=", + "owner": "rensa-nix", + "repo": "devshell", + "rev": "7d0c4bc78d9f017a739b0c7eb2f4e563118353e6", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "rensa-nix", + "repo": "devshell", + "type": "gitlab" + } + }, + "nixmkdocs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1763481845, + "narHash": "sha256-Bp0+9rDmlPWMcnKqGx+BG4+o5KO8FuDAOvXRnXrm3Fo=", + "owner": "TECHNOFAB", + "repo": "nixmkdocs", + "rev": "73d59093df94a894d25bc4bf71880b6f00faa62f", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "TECHNOFAB", + "repo": "nixmkdocs", + "type": "gitlab" + } + }, + "nixtest-lib": { + "locked": { + "dir": "lib", + "lastModified": 1759340550, + "narHash": "sha256-EH9heYb/nHHzCpUGQGqVQnuyVGQ7D6MVMgJmzNvvmJ8=", + "owner": "TECHNOFAB", + "repo": "nixtest", + "rev": "5a7053afcbb211b9cf8fe87f7892bb9f6b76b678", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "TECHNOFAB", + "repo": "nixtest", + "type": "gitlab" + } + }, + "root": { + "inputs": { + "devshell-lib": "devshell-lib", + "nixmkdocs-lib": "nixmkdocs-lib", + "nixtest-lib": "nixtest-lib", + "soonix-lib": "soonix-lib", + "treefmt-nix": "treefmt-nix" + } + }, + "soonix-lib": { + "locked": { + "dir": "lib", + "lastModified": 1763323017, + "narHash": "sha256-MJyg37d+VMfRoFiVUj16FW+zkEwQXbgK9LoFF/SHoxA=", + "owner": "TECHNOFAB", + "repo": "soonix", + "rev": "078034b01e4eaf1f9436d46721f7cbe0d96eb8b4", + "type": "gitlab" + }, + "original": { + "dir": "lib", + "owner": "TECHNOFAB", + "repo": "soonix", + "type": "gitlab" + } + }, + "treefmt-nix": { + "flake": false, + "locked": { + "lastModified": 1762938485, + "narHash": "sha256-AlEObg0syDl+Spi4LsZIBrjw+snSVU4T8MOeuZJUJjM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "5b4ee75aeefd1e2d5a1cc43cf6ba65eba75e83e4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/repo/flake.nix b/nix/repo/flake.nix new file mode 100644 index 0000000..149ae74 --- /dev/null +++ b/nix/repo/flake.nix @@ -0,0 +1,22 @@ +{ + inputs = { + devshell-lib.url = "gitlab:rensa-nix/devshell?dir=lib"; + nixtest-lib.url = "gitlab:TECHNOFAB/nixtest?dir=lib"; + soonix-lib.url = "gitlab:TECHNOFAB/soonix?dir=lib"; + nixmkdocs-lib.url = "gitlab:TECHNOFAB/nixmkdocs?dir=lib"; + treefmt-nix = { + url = "github:numtide/treefmt-nix"; + flake = false; + }; + }; + outputs = i: + i + // { + devshell = i.devshell-lib.lib {inherit (i.parent) pkgs;}; + soonix = i.soonix-lib.lib {inherit (i.parent) pkgs;}; + ntlib = i.nixtest-lib.lib {inherit (i.parent) pkgs;}; + doclib = i.nixmkdocs-lib.lib {inherit (i.parent) pkgs;}; + cilib = import "${i.parent.self}/lib" {inherit (i.parent) pkgs;}; + treefmt = import i.treefmt-nix; + }; +} diff --git a/nix/repo/tests.nix b/nix/repo/tests.nix new file mode 100644 index 0000000..62c0833 --- /dev/null +++ b/nix/repo/tests.nix @@ -0,0 +1,10 @@ +{inputs, ...}: let + inherit (inputs) pkgs ntlib cilib; +in { + tests = ntlib.mkNixtest { + modules = ntlib.autodiscover {dir = "${inputs.self}/tests";}; + args = { + inherit pkgs ntlib cilib; + }; + }; +} diff --git a/templates/nix-gitlab-ci.yml b/templates/nix-gitlab-ci.yml index 627d15c..57c56d9 100644 --- a/templates/nix-gitlab-ci.yml +++ b/templates/nix-gitlab-ci.yml @@ -1,64 +1,114 @@ spec: inputs: - image_tag: + cache_strategy: type: string - description: "latest | latest-cachix | latest-attic etc." - default: latest + description: | + auto (default) | none | runner | cachix | attic + Sets the default caching strategy. + - "auto": dynamically selects the best strategy for every job based on env variables + - "none": disables caching + - "runner", "cachix" & "attic": forces every job to use this strategy + + Can be overridden by setting NIX_CI_CACHE_STRATEGY in the pipeline variables. + default: "auto" cache_files: type: array description: | Files to use as the cache key for the generated pipeline yaml. - If you use "ci.nix" to define CI, add that here for example + If you use "ci.nix" to define CI, add that here for example. + Note that max 2 items are allowed in cache:key:files, so use something like + ["flake.*", "ci.nix"] f. ex. to match flake.lock, flake.nix and ci.nix. default: ["flake.nix", "flake.lock"] - disable_cache: + version: type: string description: | - Disables any caching provided by this component. Set to any non-empty value to disable caching. - default: "" + Which version of the Nix CI image to use. Using a tag/version is recommended. + stage_build: + type: string + description: The CI stage for building the dynamic pipeline. + default: build + stage_trigger: + type: string + description: The CI stage for triggering the dynamic pipeline. + default: trigger --- stages: - - build - - trigger + - $[[ inputs.stage_build ]] + - $[[ inputs.stage_trigger ]] variables: - NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]${NIX_CI_DISABLE_CACHE:-}" + # These can be overriden, see https://docs.gitlab.com/ci/variables/#cicd-variable-precedence + # which image should be used by default. + NIX_CI_IMAGE: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.version ]] + # default cache stategy + NIX_CI_CACHE_STRATEGY: $[[ inputs.cache_strategy ]] nix-ci:build: - stage: build - image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]] + stage: $[[ inputs.stage_build ]] + image: $NIX_CI_IMAGE cache: - key: files: $[[ inputs.cache_files ]] paths: - - generated-gitlab-ci.yml + - .nix-ci-pipelines/ - key: nix paths: - .nix-cache/ before_script: - # generated-gitlab-ci.yml exists in the cache - - '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "Using cached pipeline file (skip cache with NIX_CI_SKIP_CACHE)" || true' - # allow the user to manually skip the cache (when the key files are not correctly configured etc.) - - '[ -n "$NIX_CI_SKIP_CACHE" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_SKIP_CACHE)" || true' - # only setup when we need to generate the pipeline yaml - - 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi' + - | + # if no explicit pipeline is requested + if [[ -z "${NIX_CI_PIPELINE_NAME:-}" ]]; then + # if regex matches, use pipeline "default", otherwise $CI_PIPELINE_SOURCE + [[ "${CI_PIPELINE_SOURCE}" =~ ${NIX_CI_DEFAULT_SOURCES:-.*} ]] \ + && NIX_CI_PIPELINE_NAME="default" \ + || NIX_CI_PIPELINE_NAME="$CI_PIPELINE_SOURCE"; + fi + echo "NIX_CI_GENERATED_PIPELINE_NAME=$NIX_CI_PIPELINE_NAME" >> trigger.env + echo "ORIGINAL_CI_PIPELINE_SOURCE=$CI_PIPELINE_SOURCE" >> trigger.env + # inheritance of pipeline variables is a bit weird, so explicitly override them + # (ctx: setting any of these in the project variables would only apply correctly + # in this pipeline, not the child pipeline, instead weirdly enough the default + # variables above are used). If any other variables are added at the top, add them + # here aswell + echo "NIX_CI_IMAGE=$NIX_CI_IMAGE" >> trigger.env + echo "NIX_CI_CACHE_STRATEGY=$NIX_CI_CACHE_STRATEGY" >> trigger.env + + mkdir -p .nix-ci-pipelines/ + # generated-gitlab-ci.yml exists in the cache + [[ -f ".nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml" ]] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true + # allow the user to manually skip the cache (when the key files are not correctly configured etc.) + [[ -n "$NIX_CI_FORCE_BUILD" ]] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true + + # only setup when we need to generate the pipeline yaml + if [[ -z "$CACHED" ]]; then + source setup_nix_ci; + fi script: - # build the generated-gitlab-ci.yml if it does not exist in the cache - - 'if [ -z "$CACHED" ]; then nix build .#gitlab-ci-config && install result generated-gitlab-ci.yml; fi' + # build the pipeline if it does not exist in the cache + - > + if [[ -z "$CACHED" ]]; then + nix build .#gitlab-ci:pipeline:${NIX_CI_PIPELINE_NAME} && install result .nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml; + fi after_script: - # NOTE: environment variables of before_script and script don't exist here anymore - # # save to binary cache or Gitlab CI cache only if we actually built something # check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called - - 'if [ -f "/tmp/nix-store-before" ]; then finalize_nix_ci; fi' + - | + if [[ -f "/tmp/nix-store-before" ]]; then + finalize_nix_ci; + fi artifacts: paths: - - generated-gitlab-ci.yml + - .nix-ci-pipelines/ + reports: + dotenv: trigger.env + nix-ci:trigger: - stage: trigger + stage: $[[ inputs.stage_trigger ]] needs: - nix-ci:build trigger: include: - - artifact: generated-gitlab-ci.yml + - artifact: .nix-ci-pipelines/${NIX_CI_GENERATED_PIPELINE_NAME}.yml job: nix-ci:build strategy: depend forward: pipeline_variables: true + diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix new file mode 100644 index 0000000..3b76fb8 --- /dev/null +++ b/tests/cilib_test.nix @@ -0,0 +1,209 @@ +{ + pkgs, + cilib, + ntlib, + ... +}: { + suites."CI Lib" = { + pos = __curPos; + tests = let + inherit (cilib) mkJobDeps mkJobRun mkJobPatched mkPipeline; + deps = mkJobDeps { + key = "test"; + job = { + variables.TEST = "${pkgs.curl}"; + }; + nixConfig = { + enable = true; + deps = [pkgs.hello]; + }; + }; + in [ + { + name = "jobDeps"; + type = "script"; + script = + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${deps} "/nix/store" "should contain nix store path" + assert_file_contains ${deps} '-hello-.*/bin:$PATH' "should contain hello" + assert_file_contains ${deps} "export TEST=" "should export TEST" + assert_file_contains ${deps} "curl" "should contain curl" + ''; + } + { + name = "jobRun"; + type = "script"; + script = let + run = mkJobRun { + key = "test"; + job.script = ["hello"]; + jobDeps = deps; + }; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${run}/bin/gitlab-ci-job:test "sandbox-helper" "should contain sandbox-helper" + assert_file_contains ${run}/bin/gitlab-ci-job:test "gitlab-ci-job-test-raw" "should contain job name" + assert_file_contains ${run.passthru.actualJobScript} "gitlab-ci-job-deps-test" "should contain job name" + assert_file_contains ${run.passthru.actualJobScript} "Running script..." "should contain 'Running script...'" + assert_file_contains ${run.passthru.actualJobScript} "hello" "should contain hello" + ''; + } + { + name = "jobPatched nix disabled"; + expected = {}; + actual = mkJobPatched { + key = "test"; + pipelineName = "test"; + job = {}; + nixConfig.enable = false; + }; + } + { + name = "jobPatched nix disabled with variables and cache"; + expected = { + variables."HELLO" = "world"; + cache = [{key = "example";}]; + }; + actual = mkJobPatched { + key = "test"; + pipelineName = "test"; + job = { + variables."HELLO" = "world"; + cache = [{key = "example";}]; + }; + nixConfig.enable = false; + }; + } + { + name = "jobPatched without runner cache"; + expected = { + after_script = ["finalize_nix_ci"]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + }; + actual = mkJobPatched { + key = "test"; + pipelineName = "test"; + job = {}; + nixConfig = { + enable = true; + enableRunnerCache = false; + }; + }; + } + { + name = "jobPatched with runner cache"; + expected = { + after_script = ["finalize_nix_ci"]; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + cache = [ + { + key = "test"; + paths = [".nix-cache/"]; + } + ]; + variables."NIX_CI_CACHE_STRATEGY" = "runner"; + }; + actual = mkJobPatched { + key = "test"; + pipelineName = "test"; + job = {}; + nixConfig = { + enable = true; + enableRunnerCache = true; + runnerCacheKey = "test"; + }; + }; + } + { + name = "mkPipeline empty"; + expected = {}; + actual = + (mkPipeline { + name = "test"; + nixConfig = {}; + pipeline.jobs = {}; + }).finalConfig; + } + { + name = "mkPipeline empty packages"; + type = "script"; + script = let + pipeline = + ntlib.helpers.toJsonFile + (mkPipeline { + name = "test"; + nixConfig = {}; + pipeline.jobs = {}; + }).packages; + in + # sh + '' + set -euo pipefail + ${ntlib.helpers.path (with pkgs; [jq gnugrep coreutils])} + echo "two keys, one json one pretty" + jq 'keys | length == 2' "${pipeline}" | grep -q true + echo "key[0] is exactly 'gitlab-ci:pipeline:test'" + jq -r 'keys[0]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test" + echo "key[1] is exactly 'gitlab-ci:pipeline:test:pretty'" + jq -r 'keys[1]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test:pretty" + echo "value contains '/nix/store/'" + jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "/nix/store/" + echo "value contains 'gitlab-ci-test.yml'" + jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "gitlab-ci-test.yml" + echo "file only contains '{}'" + [[ "$(cat $(jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}"))" == "{}" ]] + ''; + } + { + name = "ignore store paths in variables with nix disabled"; + expected = { + stages = ["test"]; + test = { + stage = "test"; + variables."TEST" = "${pkgs.hello}"; + }; + }; + actual = + (mkPipeline { + name = "test"; + nixConfig.enable = false; + pipeline = { + stages = ["test"]; + jobs.test = { + stage = "test"; + variables."TEST" = "${pkgs.hello}"; + }; + }; + }).finalConfig; + } + { + # it doesn't make much sense to have any nix store path in variables, but we ignore it for global variables + name = "ignore store paths in global variables"; + expected = { + variables = { + HELLO = "world"; + CURL = toString pkgs.curl; + }; + }; + actual = + (mkPipeline { + name = "test"; + nixConfig.enable = true; + pipeline = { + variables = { + HELLO = "world"; + CURL = toString pkgs.curl; + }; + jobs = {}; + }; + }).finalConfig; + } + ]; + }; +} diff --git a/tests/fixtures/flake_parts/flake.lock b/tests/fixtures/flake_parts/flake.lock new file mode 100644 index 0000000..15b9835 --- /dev/null +++ b/tests/fixtures/flake_parts/flake.lock @@ -0,0 +1,77 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1754487366, + "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1756636162, + "narHash": "sha256-mBecwgUTWRgClJYqcF+y4O1bY8PQHqeDpB+zsAn+/zA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "37ff64b7108517f8b6ba5705ee5085eac636a249", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1753579242, + "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "systems": "systems" + } + }, + "systems": { + "locked": { + "lastModified": 1689347949, + "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", + "owner": "nix-systems", + "repo": "default-linux", + "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default-linux", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/tests/fixtures/flake_parts/flake.nix b/tests/fixtures/flake_parts/flake.nix new file mode 100644 index 0000000..cca696f --- /dev/null +++ b/tests/fixtures/flake_parts/flake.nix @@ -0,0 +1,41 @@ +{ + outputs = { + flake-parts, + systems, + ... + } @ inputs: + flake-parts.lib.mkFlake {inherit inputs;} { + imports = [ + "@repo_path@/lib/flakeModule.nix" + ]; + systems = import systems; + flake = {}; + perSystem = _: { + ci = { + config = { + # true is already default, just for testing + nixJobsByDefault = true; + }; + pipelines = { + "default" = { + stages = ["example"]; + jobs."example" = { + stage = "example"; + script = ["echo hello world"]; + }; + }; + "test".jobs."example" = { + stage = ".pre"; + script = ["echo hello world"]; + }; + }; + }; + }; + }; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + systems.url = "github:nix-systems/default-linux"; + }; +} diff --git a/tests/flake_parts_test.nix b/tests/flake_parts_test.nix new file mode 100644 index 0000000..e998f45 --- /dev/null +++ b/tests/flake_parts_test.nix @@ -0,0 +1,38 @@ +{ + pkgs, + ntlib, + ... +}: { + suites."flake-parts" = { + pos = __curPos; + tests = [ + { + name = "flakeModule"; + type = "script"; + script = + # sh + '' + ${ntlib.helpers.scriptHelpers} + ${ntlib.helpers.path (with pkgs; [coreutils nix gnused gnugrep jq])} + export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt + export NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt + repo_path=${../.} + + cp ${./fixtures/flake_parts}/* . + # import from the absolute path above, is easier than trying to figure out the repo path etc. + sed -i -e "s|@repo_path@|$repo_path|" flake.nix + + # NOTE: --impure is required since importing modules from absolute paths is not allowed in pure mode + nix build --impure .#gitlab-ci:pipeline:default + assert "-f result" "should exist" + assert_file_contains "result" "finalize_nix_ci" + jq '.' result # check if valid json just to be sure + + nix build --impure .#gitlab-ci:pipeline:default:pretty + assert "-f result" "should exist" + assert_file_contains "result" " - finalize_nix_ci" + ''; + } + ]; + }; +} diff --git a/tests/helpers_test.nix b/tests/helpers_test.nix new file mode 100644 index 0000000..4d7e7d4 --- /dev/null +++ b/tests/helpers_test.nix @@ -0,0 +1,68 @@ +{ + pkgs, + cilib, + ... +}: { + suites."Helpers" = { + pos = __curPos; + tests = let + inherit (cilib) helpers; + in [ + { + name = "appendToAfterScript"; + expected = { + after_script = ["echo after_script" "finalize_nix_ci"]; + }; + actual = helpers.appendToAfterScript ["finalize_nix_ci"] { + after_script = ["echo after_script"]; + }; + } + { + name = "prependToBeforeScript"; + expected = { + before_script = ["setup_nix_ci" "echo before_script"]; + }; + actual = helpers.prependToBeforeScript ["setup_nix_ci"] { + before_script = ["echo before_script"]; + }; + } + { + name = "toYaml"; + expected = ''{"hello":"world"}''; + actual = builtins.readFile (helpers.toYaml "test" {hello = "world";}); + } + { + name = "filterAttrsRec"; + expected = {world = "world";}; + actual = helpers.filterAttrsRec (_n: v: v != null) { + hello = null; + world = "world"; + }; + } + { + name = "filterJobVariables with store paths"; + expected = { + HELLO = "${pkgs.hello}"; + MULTIPLE = "${pkgs.hello}:${pkgs.hello}"; + }; + actual = helpers.filterJobVariables true { + variables = { + HELLO = "${pkgs.hello}"; + WORLD = "world"; + MULTIPLE = "${pkgs.hello}:${pkgs.hello}"; + }; + }; + } + { + name = "filterJobVariables without store paths"; + expected = {WORLD = "world";}; + actual = helpers.filterJobVariables false { + variables = { + HELLO = "${pkgs.hello}"; + WORLD = "world"; + }; + }; + } + ]; + }; +} diff --git a/tests/modules_test.nix b/tests/modules_test.nix new file mode 100644 index 0000000..8ccbd83 --- /dev/null +++ b/tests/modules_test.nix @@ -0,0 +1,103 @@ +{ + pkgs, + cilib, + ntlib, + ... +}: { + suites."Modules" = { + pos = __curPos; + tests = let + simplePipeline = cilib.mkCI { + pipelines."test" = { + stages = ["test"]; + jobs."test" = { + stage = "test"; + script = ["echo hello world"]; + }; + }; + }; + in [ + { + name = "empty pipelines"; + expected = {}; + actual = + (cilib.mkCI {}).pipelines; + } + { + name = "empty packages"; + expected = {}; + actual = + (cilib.mkCI {}).packages; + } + { + name = "simple pipeline"; + expected = { + stages = [".pre" "test" ".post"]; + "test" = { + image = "$NIX_CI_IMAGE"; + stage = "test"; + before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; + script = ["echo hello world"]; + after_script = ["finalize_nix_ci"]; + }; + }; + actual = simplePipeline.pipelines."test".finalConfig; + } + { + name = "simple pipeline yaml"; + type = "script"; + script = let + package = simplePipeline.packages."gitlab-ci:pipeline:test"; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${package} 'gitlab-ci:pipeline:test:job-deps:test' + assert_file_contains ${package} 'finalize_nix_ci' + assert_file_contains ${package} 'echo hello world' + ''; + } + { + name = "dont fail on store paths"; + type = "script"; + script = let + package = + (cilib.mkCI { + pipelines."test" = { + variables.EXAMPLE = "${pkgs.hello}"; + }; + }).packages."gitlab-ci:pipeline:test"; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${package} '[".pre",".post"]' + assert_file_contains ${package} '"EXAMPLE":"/nix/store/.*-hello-.*"' + ''; + } + { + name = "correctly inject variables containing nix store paths at runtime"; + type = "script"; + script = let + package = + (cilib.mkCI { + pipelines."test".jobs."test" = { + stage = ".pre"; + variables.EXAMPLE = "${pkgs.hello}"; + script = []; + }; + }).packages."gitlab-ci:pipeline:test:job-deps:test"; + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${package} 'export PATH=":$PATH";' + assert_file_contains ${package} 'export EXAMPLE="/nix/store/.*-hello-.*"' + ''; + } + ]; + }; +} diff --git a/tests/soonix_test.nix b/tests/soonix_test.nix new file mode 100644 index 0000000..c76edb4 --- /dev/null +++ b/tests/soonix_test.nix @@ -0,0 +1,77 @@ +{ + lib, + cilib, + ... +}: let + inherit (lib) trimWith; +in { + suites."Soonix" = { + pos = __curPos; + tests = let + version = trimWith { + start = true; + end = true; + } (builtins.readFile ../lib/VERSION); + in [ + { + name = "default soonix config"; + expected = { + data.include = [ + { + component = "gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@${version}"; + inputs.version = version; + } + ]; + generator = "nix"; + opts.format = "yaml"; + hook = { + mode = "copy"; + gitignore = false; + }; + output = ".gitlab-ci.yml"; + }; + actual = + (cilib.mkCI { + config.soonix = {}; + }).soonix; + } + { + name = "custom soonix config"; + expected = { + data = { + include = [ + { + component = "gitlab.com/example/nix-gitlab-ci/nix-gitlab-ci@abc"; + inputs = { + version = "abc"; + hello = "world"; + }; + } + ]; + "example".script = ["hello"]; + }; + generator = "nix"; + opts.format = "yaml"; + hook = { + mode = "copy"; + gitignore = false; + }; + output = ".gitlab-ci.yml"; + }; + actual = + (cilib.mkCI { + config.soonix = { + componentVersion = "abc"; + componentUrl = "gitlab.com/example/nix-gitlab-ci/nix-gitlab-ci"; + componentInputs = { + hello = "world"; + }; + extraData = { + "example".script = ["hello"]; + }; + }; + }).soonix; + } + ]; + }; +} diff --git a/tests/utils_test.nix b/tests/utils_test.nix new file mode 100644 index 0000000..2e319b1 --- /dev/null +++ b/tests/utils_test.nix @@ -0,0 +1,32 @@ +{ + pkgs, + ntlib, + cilib, + ... +}: { + suites."Utils" = { + pos = __curPos; + tests = [ + { + name = "commitAndPushFiles"; + type = "script"; + script = let + inherit (cilib) utils; + job = ntlib.helpers.toJsonFile ( + utils.commitAndPushFiles { + message = "hello world"; + files = ["a.md" "b.txt"]; + } {} + ); + in + # sh + '' + ${ntlib.helpers.path [pkgs.gnugrep]} + ${ntlib.helpers.scriptHelpers} + assert_file_contains ${job} 'git commit -m \\"hello world\\"' + assert_file_contains ${job} 'git add a.md b.txt' + ''; + } + ]; + }; +}