diff --git a/.envrc b/.envrc index 565a52a..a2d9328 100644 --- a/.envrc +++ b/.envrc @@ -1,2 +1,4 @@ -source $(fetchurl https://gitlab.com/rensa-nix/direnv/-/raw/v0.3.0/direnvrc "sha256-u7+KEz684NnIZ+Vh5x5qLrt8rKdnUNexewBoeTcEVHQ=") -use ren //repo/devShells/default +if ! use flake . --impure +then + echo "devenv could not be build. The devenv environment was not loaded. Make the necessary changes to flake.nix and hit enter to try again." >&2 +fi diff --git a/.gitignore b/.gitignore index fe07db9..2a8dde4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ -.ren/ +.idea +.devenv +.direnv +.pre-commit-config.yaml result diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8c146c..b0393fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,59 +1,35 @@ -# Generated by soonix, DO NOT EDIT +include: + - component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHA + inputs: + image_tag: $CI_COMMIT_SHORT_SHA +stages: + - build-images + - build + - trigger build:image: - after_script: - - install -D result dist/nix-ci-$ARCH.tar.gz - artifacts: - paths: - - dist - image: nixpkgs/nix-flakes:latest + stage: build-images parallel: matrix: - - ARCH: - - x86_64-linux - - aarch64-linux - script: - - nix build .#image --system $ARCH - stage: build-images -deploy:image: - before_script: - - 'nix profile install nixpkgs#buildah - - export PATH="$PATH:$HOME/.nix-profile/bin" - - export REGISTRY_AUTH_FILE=${HOME}/auth.json - - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin - $CI_REGISTRY - - mkdir -p /etc/containers && echo ''{"default":[{"type":"insecureAcceptAnything"}]}'' - > /etc/containers/policy.json - - mkdir -p /var/tmp - - ' + - VARIANT: ["", "-cachix", "-attic"] image: nixpkgs/nix-flakes:latest - needs: - - build:image + before_script: + - nix profile install nixpkgs#skopeo + - export PATH="$PATH:$HOME/.nix-profile/bin" script: - - "export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\\//-}\nbuildah manifest create localhost/nix-ci\n\ - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz\n\ - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz\n\ - buildah manifest push --all localhost/nix-ci docker://${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_SHORT_SHA}\n\ - # branches\nif [ -z \"$CI_COMMIT_TAG\" ]; then\n buildah manifest push --all\ - \ localhost/nix-ci docker://${CI_REGISTRY_IMAGE}/nix-ci:${NORMALIZED_BRANCH/main/latest}\n\ - fi\n# tags\nif [ -n \"$CI_COMMIT_TAG\" ]; then\n buildah manifest push --all\ - \ localhost/nix-ci docker://${CI_REGISTRY_IMAGE}/nix-ci:${CI_COMMIT_TAG}\nfi\n" - stage: build-images -include: -- component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci@$CI_COMMIT_SHORT_SHA - inputs: - cache_files: - - flake.* - - nix/repo/ci.nix - version: $CI_COMMIT_SHORT_SHA -stages: -- build-images -- build -- trigger -variables: - NIX_CI_IMAGE: $CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA + - nix build .#image${VARIANT} + - export NORMALIZED_BRANCH=${CI_COMMIT_BRANCH/\//-} + - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp "docker-archive:result" "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_SHORT_SHA}${VARIANT}" + # branches + - | + if [ -z "$CI_COMMIT_TAG" ]; then + skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ + "docker-archive:result" \ + "docker://$CI_REGISTRY_IMAGE/nix-ci:${NORMALIZED_BRANCH/main/latest}${VARIANT}"; + fi + # tags + - | + if [ -n "$CI_COMMIT_TAG" ]; then + skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" --tmpdir /tmp \ + "docker-archive:result" \ + "docker://$CI_REGISTRY_IMAGE/nix-ci:${CI_COMMIT_TAG}${VARIANT}"; + fi diff --git a/README.md b/README.md index c5e6b92..4b24815 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,4 @@ -# Nix GitLab CI - -[![built with nix](https://img.shields.io/static/v1?logo=nixos&logoColor=white&label=&message=Built%20with%20Nix&color=41439a)](https://builtwithnix.org) -[![pipeline status](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/badges/main/pipeline.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/commits/main) -![License: MIT](https://img.shields.io/gitlab/license/technofab/nix-gitlab-ci) -[![Latest Release](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/badges/release.svg)](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/releases) -[![Support me](https://img.shields.io/badge/Support-me-orange)](https://tec.tf/#support) -[![Docs](https://img.shields.io/badge/Read-Docs-orange)](https://nix-gitlab-ci.projects.tf) +# Nix Gitlab CI Flake module which allows generating a `.gitlab-ci.yml` from Nix. @@ -13,13 +6,13 @@ This allows easily using any Nix package in CI. Also makes it possible to split CI parts in a separate module which can be imported in multiple projects. -## Usage (with flake-parts) +## Usage ```nix # flake.nix { ... - inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; # recommendation: pin to the latest release/version + inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; outputs = {...}: flake-parts.lib.mkFlake {...} { imports = [ @@ -29,26 +22,16 @@ Also makes it possible to split CI parts in a separate module which can be impor perSystem = {pkgs, ...}: { ci = { - config = { - # configure Nix-GitLab-CI here, see docs for options - }; - pipelines."default" = { - stages = ["test"]; - jobs = { - "test" = { - stage = "test"; - nix.deps = [pkgs.unixtools.ping]; - script = [ - "ping -c 5 8.8.8.8" - ]; - }; + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + nix.deps = [pkgs.unixtools.ping]; + script = [ + "ping -c 5 8.8.8.8" + ]; }; }; - # runs on a merge request for example - pipelines."merge_request_event" = { - stages = ["some_stage"]; - jobs = { ... }; - }; }; ... } @@ -56,28 +39,13 @@ Also makes it possible to split CI parts in a separate module which can be impor } ``` -Now either use this in your .gitlab-ci.yml or setup Soonix to auto generate this -file for you with the right version (see the [docs][docs-soonix] for more). - ```yaml # .gitlab-ci.yml include: - - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: pin to the latest release/version (don't use "main" etc.) + - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@1.1.0 inputs: - version: # docker image tag, use the same version as a above -``` - -## Usage (directly) - -```nix -let - cilib = inputs.nix-gitlab-ci.lib {inherit pkgs;}; -in - cilib.mkCI { - config = ...; - pipelines."default" = ...; - }; - # exposes `soonix` for the soonix hook and `packages` which contain the configs, jobs etc. + # specify inputs here, for example: + image_tag: latest-cachix ``` ## Utilities @@ -87,25 +55,24 @@ in To disable any of the provided caches for a pipeline one can set `NIX_CI_DISABLE_CACHE` to anything non-empty (eg. "yes") when triggering the pipeline. -The `build:nix-ci` job has a different special environment variable `NIX_CI_FORCE_BUILD` -(useful if the generated pipeline in the cache is outdated, this will build it again). +The `build:nix-ci` job has a different special environment variable `NIX_CI_SKIP_CACHE` +(useful if the generated pipeline is outdated but caching should generally still take place). ### Run Jobs locally You can run any job's script (+ before and after) locally with Nix for easier testing: ```sh -# / pipeline name, like "default" -nix run .#gitlab-ci:pipeline::job: +nix run .#gitlab-ci-job: ``` -There is also `.#gitlab-ci:pipeline::job-deps:` which generates and exports the required environment variables for each job: +There is also `.#gitlab-ci-job-deps:` which generates and exports the required environment variables for each job: - PATH (with all deps) - any custom env variables which contain store paths to not break stuff when switching archs +Please see #8 for some issues and further improvements on this. + ## Thanks to Some parts of this implementation are adapted/inspired from https://gitlab.com/Cynerd/gitlab-ci-nix - -[docs-soonix]: https://nix-gitlab-ci.projects.tf/soonix "Soonix Integration" diff --git a/docs/caching.md b/docs/caching.md deleted file mode 100644 index 0a84110..0000000 --- a/docs/caching.md +++ /dev/null @@ -1,50 +0,0 @@ -# Caching - -Nix GitLab CI supports several caching mechanisms to speed up your pipelines. - -## GitLab Runner Cache - -The runner cache strategy copies the new store paths into a directory `.nix-cache`, -which is then saved in the regular GitLab cache (technically runner cache). -It's also configured as a substituter automatically. - -To enable, set the cache strategy to `runner`. - -Configure it using these environment variables: - -- `RUNNER_CACHE`: path to the runner cache (default `.nix-cache`) - -!!! warning - - This is very inefficient and should probably only be used for very very small - dependency counts. Otherwise it takes an eternity to save to cache. - -## Cachix - -Cachix is a hosted binary cache service that can significantly speed up Nix -builds by sharing build results. - -To enable, set the cache strategy to `attic`. - -Configure it using these environment variables: - -- `CACHIX_CACHE`: name of the cache to use -- (`CACHIX_AUTH_TOKEN`): cachix client itself uses this for authentication - -!!! warning - - Cachix has not been tested. Feedback is appreciated :) - -## Attic (Self-Hosted Cache) - -Attic is a self-hosted, deduplicating binary cache. It's a great option if you -want more control over your caching infrastructure and to have the cache closer -to your runners. - -To enable, set the cache strategy to `attic`. - -Configure it using these environment variables: - -- `ATTIC_SERVER`: URL of the server -- `ATTIC_CACHE`: name of the cache to use -- `ATTIC_TOKEN`: auth token from the attic server diff --git a/docs/cicd_component.md b/docs/cicd_component.md deleted file mode 100644 index 4ea3b17..0000000 --- a/docs/cicd_component.md +++ /dev/null @@ -1,48 +0,0 @@ -# CI/CD Component - -The CI/CD Component has some inputs which configure defaults for Nix GitLab CI. - -!!! WARNING - - If you get errors like `the component path is not supported` it might be related to - [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/437996#note_1775337668). - - See [here](https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/issues/27) for more. - -## `version` - -- Type: `string` - -Which version of the Nix CI image to use. Using a tag/version is recommended. -Will not do anything if a custom image is specified using `NIX_CI_IMAGE`. - -## `cache_strategy` - -- Type: `string` -- Default: `"auto"` -- Options: `auto` | `none` | `runner` | `cachix` | `attic` - -Sets the default caching strategy. - -- `auto`: dynamically selects the best strategy for every job based on env variables -- `none`: disables caching -- `runner`, `cachix` & `attic`: forces every job to use this strategy - -Can be overridden by `NIX_CI_CACHE_STRATEGY`, see [Environment Variables](./environment_variables.md). - -## `cache_files` - -- Type: `array` (of strings) -- Default: `["flake.nix", "flake.lock"]` - -Files to use as the cache key for the generated pipeline yaml. -If you use a file like `ci.nix` to define CI, add that here for example. -This makes sure that changes to your Nix CI configuration will invalidate the cache, -otherwise an old pipeline yaml might be used. - -!!! warning - - The value of this is used in `cache:key:files`, which currently only supports - a max of 2 entries. So use something like `["flake.*", "ci.nix"]` to match - `flake.lock`, `flake.nix` and `ci.nix`. - See [gitlab-org/gitlab#301161](https://gitlab.com/gitlab-org/gitlab/-/issues/301161) diff --git a/docs/environment_variables.md b/docs/environment_variables.md deleted file mode 100644 index 56c0ba5..0000000 --- a/docs/environment_variables.md +++ /dev/null @@ -1,107 +0,0 @@ -# Environment Variables - -Nix GitLab CI is mostly controlled using environment variables. -This page outlines all the variables and their use case. - -## `NIX_CI_IMAGE` - -| | | -| ----------- | -------------------------------------------------------------------------- | -| Default | `registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci@$[[ inputs.version ]]` | -| Description | Image to use for the jobs | - -## `NIX_CI_PIPELINE_NAME` - -| | | -| ----------- | ------------------------------------------------- | -| Default | N/A | -| Description | Explicitly request a pipeline to be built and ran | -| See also | [Multi Pipeline](./multi_pipeline.md) | - -## `NIX_CI_DEFAULT_SOURCES` - -| | | -| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | -| Default | `.*` | -| Description | Regex to match `$CI_PIPELINE_SOURCE` against. If it matches, the `default` pipeline will be ran, otherwise `$CI_PIPELINE_SOURCE` | -| See also | [Multi Pipeline](./multi_pipeline.md) | - -## `NIX_CI_FORCE_BUILD` - -| | | -| ----------- | -------------------------------------------------------------------------------------- | -| Default | N/A | -| Description | Set to any non-empty value to force the `nix-ci:build` job to freshly build the config | -| See also | [Caching](./caching.md) | - -## `NIX_CI_DISABLE_CACHE` - -| | | -| ----------- | ------------------------------------------------------ | -| Default | N/A | -| Description | Set to any non-empty value to disable caching for jobs | -| See also | [Caching](./caching.md) | - -## `NIX_CI_CACHE_STRATEGY` - -| | | -| ----------- | --------------------------------------------------------------------------------- | -| Default | `$[[ inputs.cache_strategy ]]` -> defaults to `auto` | -| Description | Caching strategy to use. `auto` will select the strategy based on runner settings | -| See also | [Caching](./caching.md) | - -## `NIX_CI_RUNNER_CACHE_STRATEGY` - -| | | -| ----------- | ---------------------------------------------------------------- | -| Default | N/A | -| Description | Every runner can set it's own preferred cache strategy with this | -| See also | [Caching](./caching.md) | - -## `NIX_CI_DEFAULT_CACHE_STRATEGY` - -| | | -| ----------- | ------------------------------------------------------------------------------------------------- | -| Default | `none` | -| Description | If no runner cache strategy is set and the main strategy is set to auto, this will be the default | -| See also | [Caching](./caching.md) | - -## `RUNNER_CACHE` - -| | | -| ----------- | -------------------------------------- | -| Default | `.nix-cache` | -| Description | Path to directory for the runner cache | -| See also | [Caching](./caching.md) | - -## `CACHIX_CACHE` - -| | | -| ----------- | ------------------------------- | -| Default | N/A | -| Description | Name of the cachix cache to use | -| See also | [Caching](./caching.md) | - -## `ATTIC_CACHE` - -| | | -| ----------- | ------------------------------ | -| Default | N/A | -| Description | Name of the attic cache to use | -| See also | [Caching](./caching.md) | - -## `ATTIC_SERVER` - -| | | -| ----------- | ----------------------- | -| Default | N/A | -| Description | URL of the attic server | -| See also | [Caching](./caching.md) | - -## `ATTIC_TOKEN` - -| | | -| ----------- | ------------------------------- | -| Default | N/A | -| Description | API token from the attic server | -| See also | [Caching](./caching.md) | diff --git a/docs/examples.md b/docs/examples.md deleted file mode 100644 index 336a55a..0000000 --- a/docs/examples.md +++ /dev/null @@ -1,13 +0,0 @@ -# Example Configs - -- [TECHNOFAB/nix-gitlab-ci](https://gitlab.com/TECHNOFAB/nix-gitlab-ci) -- [TECHNOFAB/nixlets](https://gitlab.com/TECHNOFAB/nixlets) -- [TECHNOFAB/nixible](https://gitlab.com/TECHNOFAB/nixible) -- [TECHNOFAB/nixmkdocs](https://gitlab.com/TECHNOFAB/nixmkdocs) -- [TECHNOFAB/tofunix](https://gitlab.com/TECHNOFAB/tofunix) -- [TECHNOFAB/nixtest](https://gitlab.com/TECHNOFAB/nixtest) - -!!! note - - Feel free to edit this page and add your project if you're using - Nix GitLab CI :) diff --git a/docs/images/logo.svg b/docs/images/logo.svg deleted file mode 100755 index d8dce95..0000000 --- a/docs/images/logo.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 14f7da3..0000000 --- a/docs/index.md +++ /dev/null @@ -1,22 +0,0 @@ -# Nix GitLab CI - -This project provides a Nix flake module that allows you to generate your `.gitlab-ci.yml` file directly from your Nix configuration. - -## Features - -- **Reproducibility:** Leverage Nix's strength in creating reproducible environments for your CI jobs. -- **Easy Dependency Management:** Easily include any package available in Nixpkgs or your own defined packages within your CI jobs using Nix. -- **Modularity:** Define and manage your CI configurations in a structured and modular way using Nix modules, making it easier to share and reuse CI logic across multiple projects. - -This documentation will guide you through setting up and using Nix GitLab CI for your projects. - -## Warnings - -To save you from frantically searching these docs if something doesn't work as expected, here are the most important warnings ;) - -!!! warning - - Do not put Nix store paths into global/pipeline variables. They will simply be passed through, - resulting in bad portability (if two runners have different archs for example, one cannot find the path). - If you need any Nix store path in env variables, always do it on the job level, there - it will automatically be computed at runtime, thus will always work no matter which runner it runs on. diff --git a/docs/kubernetes_runner.md b/docs/kubernetes_runner.md deleted file mode 100644 index 4c20421..0000000 --- a/docs/kubernetes_runner.md +++ /dev/null @@ -1,38 +0,0 @@ -# Kubernetes Runner Setup - -Using the GitLab Kubernetes runner allows your CI jobs to run as pods in a Kubernetes cluster. -Nix GitLab CI can be integrated with this setup, and using advanced configuration options like -`pod_spec` makes it easy to add runner specific caching. - -Using this Runner configuration ... - -```toml -[[runners.kubernetes.pod_spec]] -name = "nix-ci-cache-secrets" -patch = ''' - containers: - - name: build - envFrom: - - secretRef: - name: nix-ci-cache-env -''' -``` - -... and a secret containing ... - -```yaml -NIX_CI_RUNNER_CACHE_STRATEGY: attic -ATTIC_SERVER: # example: http://atticd..svc.cluster.local:8080 -ATTIC_CACHE: ci # name however you want, just needs to exist -ATTIC_TOKEN: -``` - -... makes your jobs automatically cache their Nix store paths to the in-cluster -attic when running with this runner. - -Other runners could use cachix or no cache, you get the idea ;P - -!!! note - - This of course works with any executor where you can set environment - variables. This is just an example how to do it in Kubernetes easily. diff --git a/docs/multi_pipeline.md b/docs/multi_pipeline.md deleted file mode 100644 index d8dbfe6..0000000 --- a/docs/multi_pipeline.md +++ /dev/null @@ -1,31 +0,0 @@ -# Multiple Pipelines - -With V2, Nix GitLab CI can generate different pipelines, depending on the -pipeline source (`$CI_PIPELINE_SOURCE`). - -By default, no matter which source, the `default` pipeline is built and ran. -`$NIX_CI_PIPELINE` can override that, eg. when manually triggering a run. -To configure which source should be 1-to-1 translated to a pipeline with the -same name, set `$NIX_CI_DEFAULT_SOURCES` to a regex which explicitly does not -match these sources. Or set it to an impossible to match regex, then it will -always run the pipeline named after `$CI_PIPELINE_SOURCE`. - -## Example 1: always run default - -If you only have a single pipeline, you just have to call it `default`. -Everything else works out of the box. - -## Example 2: default and merge_request_event - -If you want the source `merge_request_event` to trigger a different pipeline, -name it like that and set `$NIX_CI_DEFAULT_SOURCES` to `^(merge_request_event)$`. -Now a merge request will run this pipeline, while everything else runs `default`. - -## Example 3: default, push and web - -Set `$NIX_CI_DEFAULT_SOURCES` to `^(push|web)$`. - -## Example 4: always run the specific pipelines, never default - -Set `$NIX_CI_DEFAULT_SOURCES` to any regex that never matches the sources, -like `a\A` or `nothing`. diff --git a/docs/options.md b/docs/options.md deleted file mode 100644 index 4ca74a4..0000000 --- a/docs/options.md +++ /dev/null @@ -1,3 +0,0 @@ -# Options - -{% include 'options.md' %} diff --git a/docs/setup.md b/docs/setup.md deleted file mode 100644 index c1f85df..0000000 --- a/docs/setup.md +++ /dev/null @@ -1,81 +0,0 @@ -# Setup - -To integrate Nix GitLab CI into your project, you need to make two main changes: - -1. Add the `nix-gitlab-ci` flake module to your `flake.nix`. -1. Include the necessary component in your `.gitlab-ci.yml`. - -## Adding to `flake.nix` - -In your project's `flake.nix`, add `nix-gitlab-ci` as an input and import its -flake module within your `flake-parts` configuration. - -```nix title="flake.nix" -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; # Or your preferred nixpkgs branch/version - flake-parts.url = "github:hercules-ci/flake-parts"; - - # Add nix-gitlab-ci as an input - # recommendation: pin to a specific release/version - nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; - }; - - outputs = { nixpkgs, flake-parts, ... }@inputs: - flake-parts.lib.mkFlake { inherit inputs; } { - imports = [ - # Import the nix-gitlab-ci flake module - inputs.nix-gitlab-ci.flakeModule - ]; - - systems = [ - "x86_64-linux" - "aarch64-linux" - # Add other systems you need - ]; - - perSystem = { pkgs, ... }: { - # define your CI pipelines - # ci.pipelines."merge_request_event" = { ... }; - }; - }; -} -``` - -Replace `` with the specific version or commit hash of `nix-gitlab-ci` -you wish to use. Pinning to a specific version is highly recommended for -reproducibility and compatibility. - -!!! warning - - While the flake input is locked through `flake.lock`, the CI/CD component - will always use the latest commit of the reference. This means that by using - a branch like `main` as version for both, the CI/CD component will always use - the latest commit while your flake uses a fixed one. - This could result in drift between both, potentially breaking stuff. - -## Including in `.gitlab-ci.yml` - -Your `.gitlab-ci.yml` file will be minimal. Its primary role is to include the -`nix-gitlab-ci` component, which will then generate the full CI configuration -based on your Nix code. - -```yaml title=".gitlab-ci.yml" -include: - - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ - inputs: - # This input sets the Docker image tag used for the CI jobs. - # Use the same version as you pinned in your flake.nix for consistency. - version: -``` - -Again, ensure `` matches the version used in your `flake.nix`. -This component includes a job (`build:nix-ci`) that will evaluate your Nix -configuration and generate the `.gitlab-ci.yml` used for the pipeline run. - -!!! note - - Since V3 [Soonix](https://soonix.projects.tf) is supported, this can - automatically generate the `.gitlab-ci.yml` for you, with the version - automatically following the flake. - See [Soonix Integration](./soonix.md) for more. diff --git a/docs/soonix.md b/docs/soonix.md deleted file mode 100644 index b333ab6..0000000 --- a/docs/soonix.md +++ /dev/null @@ -1,20 +0,0 @@ -# Soonix Integration - -[Soonix](https://soonix.projects.tf) can be used to automatically generate the -`.gitlab-ci.yml` for you. -This will by default include the CI/CD component with the same version as the -flake (using the `VERSION` file in this repo). - -You can specify some options to configure this, like changing the component URL -or adding extra data to your `.gitlab-ci.yml` (like this repo does to bootstrap -Nix-GitLab-CI). See [Options](./options.md#configsoonix) for all the config options. - -You can use it like this: - -```nix -let - ci = cilib.mkCI { ... }; -in { - soonix.hooks."ci" = ci.soonix; -} -``` diff --git a/docs/style.css b/docs/style.css deleted file mode 100644 index b2ae4ff..0000000 --- a/docs/style.css +++ /dev/null @@ -1,15 +0,0 @@ -.md-header__button.md-logo { - margin: 0; - padding-top: .2rem; - padding-bottom: .2rem; -} - -[dir="ltr"] .md-header__title { - margin-left: 0; -} - -.md-header__button.md-logo img, -.md-header__button.md-logo svg { - height: 2rem; -} - diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index 44c0bf7..0000000 --- a/docs/usage.md +++ /dev/null @@ -1,73 +0,0 @@ -# Usage - -## Usage (with flake-parts) - -```nix -# flake.nix -{ - ... - inputs.nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci/?dir=lib"; # recommendation: pin to the latest release/version - - outputs = {...}: flake-parts.lib.mkFlake {...} { - imports = [ - inputs.nix-gitlab-ci.flakeModule - ]; - ... - - perSystem = {pkgs, ...}: { - ci = { - config = { - # configure Nix-GitLab-CI here, see docs for options - }; - pipelines."default" = { - stages = ["test"]; - jobs = { - "test" = { - stage = "test"; - nix.deps = [pkgs.unixtools.ping]; - script = [ - "ping -c 5 8.8.8.8" - ]; - }; - }; - }; - # runs on a merge request for example - pipelines."merge_request_event" = { - stages = ["some_stage"]; - jobs = { ... }; - }; - }; - ... - } - } -} -``` - -Now either use this in your .gitlab-ci.yml or setup Soonix to auto generate this -file for you with the right version (see the [docs][docs-soonix] for more). - -```yaml -# .gitlab-ci.yml -include: - - component: gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@ # recommendation: pin to the latest release/version (don't use "main" etc.) - inputs: - version: # docker image tag, use the same version as a above -``` - -## Usage (directly) - -```nix -let - cilib = inputs.nix-gitlab-ci.lib {inherit pkgs;}; -in - cilib.mkCI { - config = ...; - pipelines."default" = ...; - }; - # exposes `soonix` for the soonix hook and `packages` which contain the configs, jobs etc. -``` - -______________________________________________________________________ - -Since V2 multiple pipelines are supported. -See [Multiple Pipelines](./multi_pipeline.md) for more. diff --git a/docs/utilities.md b/docs/utilities.md deleted file mode 100644 index b82091d..0000000 --- a/docs/utilities.md +++ /dev/null @@ -1,64 +0,0 @@ -# Utilities - -Nix GitLab CI provides a couple of utilities to help with development and -debugging. - -## Disabling caching temporarily - -Nix GitLab CI often utilizes caching mechanisms to speed up your pipelines -(see [Caching](./caching.md)). -However, there might be situations where you need to temporarily disable these -caches for a specific pipeline run, for example, to debug a caching issue or -ensure a clean build. - -To disable most of the provided caches for a pipeline, set the environment -variable `NIX_CI_DISABLE_CACHE` to any non-empty value (e.g., `yes`, `true`, `1`) -when triggering the pipeline in the GitLab UI or via the API. - -## Forcing a rebuild of the CI pipeline definition - -The job responsible for generating the `.gitlab-ci.yml` from your Nix code -(`build:nix-ci`) might itself be cached. If you've made changes to your Nix CI -configuration and the pipeline doesn't seem to pick them up, the cached job -definition might be the reason. - -You should first double check if all the Nix files you defined the CI config in -are specified in the `cache_files` CI/CD-component input -(see [CI/CD Component](./cicd_component.md) for more). - -To force this specific job to rebuild and re-evaluate your Nix configuration, -set the environment variable `NIX_CI_FORCE_BUILD` when triggering the pipeline. - -## Running jobs locally - -One of the benefits of defining your CI jobs with Nix is the ability to run them -locally in an environment that closely mirrors the CI environment. This can -significantly speed up debugging and development. - -You can run the script of any defined job locally using the `nix run` command. -The syntax is: - -```sh -nix run .#gitlab-ci:pipeline::job: -``` - -Replace `` with the name of the pipeline the job belongs to -(e.g., `default` for jobs defined under the `ci` attribute) and `` -with the name of the job you want to run. - -This command will set up the environment with the specified `nix.deps` and -execute the job's `script`. - -There is also an attribute `.#gitlab-ci:pipeline::job-deps:`. -Building this derivation will generate a shell script which exports the required -environment variables for the job, such as the `PATH` including all dependencies -and any custom environment variables that contain store paths (ensuring they are -correctly resolved across different architectures). - -You can use this to inspect the environment that would be set up for a job without -running the full script. - -## Viewing generated config in YAML - -Since the GitLab CI config is generated simply using JSON, it's hard to read and -debug. For debugging V3 now adds another package `gitlab-ci:pipeline::pretty`. diff --git a/examples/flake-parts/flake.lock b/examples/flake-parts/flake.lock deleted file mode 100644 index 2e81bcf..0000000 --- a/examples/flake-parts/flake.lock +++ /dev/null @@ -1,95 +0,0 @@ -{ - "nodes": { - "flake-parts": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "lastModified": 1756770412, - "narHash": "sha256-+uWLQZccFHwqpGqr2Yt5VsW/PbeJVTn9Dk6SHWhNRPw=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "4524271976b625a4a605beefd893f270620fd751", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "nix-gitlab-ci": { - "locked": { - "dir": "lib", - "lastModified": 1752052838, - "narHash": "sha256-EqP4xB8YTVXWPCCchnVtQbuq0bKa79TUEcPF3hjuX/k=", - "owner": "TECHNOFAB", - "repo": "nix-gitlab-ci", - "rev": "0c6949f585a2c1ea2cf85fc01445496f7c75faae", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "TECHNOFAB", - "repo": "nix-gitlab-ci", - "type": "gitlab" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1756696532, - "narHash": "sha256-6FWagzm0b7I/IGigOv9pr6LL7NQ86mextfE8g8Q6HBg=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "58dcbf1ec551914c3756c267b8b9c8c86baa1b2f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib": { - "locked": { - "lastModified": 1754788789, - "narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "a73b9c743612e4244d865a2fdee11865283c04e6", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-parts": "flake-parts", - "nix-gitlab-ci": "nix-gitlab-ci", - "nixpkgs": "nixpkgs", - "systems": "systems" - } - }, - "systems": { - "locked": { - "lastModified": 1689347949, - "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", - "owner": "nix-systems", - "repo": "default-linux", - "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default-linux", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/examples/flake-parts/flake.nix b/examples/flake-parts/flake.nix deleted file mode 100644 index c1b7977..0000000 --- a/examples/flake-parts/flake.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ - outputs = { - flake-parts, - systems, - ... - } @ inputs: - flake-parts.lib.mkFlake {inherit inputs;} { - imports = [ - inputs.nix-gitlab-ci.flakeModule - ]; - systems = import systems; - flake = {}; - perSystem = _: { - ci = { - pipelines = { - "default" = { - stages = ["example"]; - jobs."example" = { - stage = "example"; - script = ["echo hello world"]; - }; - }; - "test".jobs."example" = { - stage = ".pre"; - script = ["echo hello world"]; - }; - }; - }; - }; - }; - - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - flake-parts.url = "github:hercules-ci/flake-parts"; - systems.url = "github:nix-systems/default-linux"; - # NOTE: better pin to a version - nix-gitlab-ci.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; - }; -} diff --git a/examples/rensa-nix/flake.lock b/examples/rensa-nix/flake.lock deleted file mode 100644 index 7c47f5c..0000000 --- a/examples/rensa-nix/flake.lock +++ /dev/null @@ -1,63 +0,0 @@ -{ - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1756542300, - "narHash": "sha256-tlOn88coG5fzdyqz6R93SQL5Gpq+m/DsWpekNFhqPQk=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d7600c775f877cd87b4f5a831c28aa94137377aa", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib": { - "locked": { - "lastModified": 1754184128, - "narHash": "sha256-AjhoyBL4eSyXf01Bmc6DiuaMrJRNdWopmdnMY0Pa/M0=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "02e72200e6d56494f4a7c0da8118760736e41b60", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, - "ren": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "dir": "lib", - "lastModified": 1756370106, - "narHash": "sha256-l84ojcHuQWBwn4BRxQsMMfQpcq/Az/sHh/hSqFgVtyg=", - "owner": "rensa-nix", - "repo": "core", - "rev": "9c1a29fa9ba7cbbb78b9e47eb8afbcd29303a3b4", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "rensa-nix", - "repo": "core", - "type": "gitlab" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs", - "ren": "ren" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/examples/rensa-nix/flake.nix b/examples/rensa-nix/flake.nix deleted file mode 100644 index 1f81895..0000000 --- a/examples/rensa-nix/flake.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - ren.url = "gitlab:rensa-nix/core?dir=lib"; - }; - - outputs = { - self, - ren, - ... - } @ inputs: - ren.buildWith - { - inherit inputs; - cellsFrom = ./nix; - transformInputs = system: i: - i - // { - pkgs = import i.nixpkgs {inherit system;}; - }; - cellBlocks = with ren.blocks; [ - (simple "ci") - ]; - } - { - packages = ren.select self [ - ["repo" "ci" "packages"] - ]; - }; -} diff --git a/examples/rensa-nix/nix/repo/ci.nix b/examples/rensa-nix/nix/repo/ci.nix deleted file mode 100644 index adb707e..0000000 --- a/examples/rensa-nix/nix/repo/ci.nix +++ /dev/null @@ -1,18 +0,0 @@ -{inputs, ...}: let - inherit (inputs) cilib; -in - cilib.mkCI { - pipelines = { - "default" = { - stages = ["example"]; - jobs."example" = { - stage = "example"; - script = ["echo hello world"]; - }; - }; - "test".jobs."example" = { - stage = ".pre"; - script = ["echo hello world"]; - }; - }; - } diff --git a/examples/rensa-nix/nix/repo/flake.lock b/examples/rensa-nix/nix/repo/flake.lock deleted file mode 100644 index fb28eb7..0000000 --- a/examples/rensa-nix/nix/repo/flake.lock +++ /dev/null @@ -1,28 +0,0 @@ -{ - "nodes": { - "nix-gitlab-ci-lib": { - "locked": { - "dir": "lib", - "lastModified": 1752052838, - "narHash": "sha256-EqP4xB8YTVXWPCCchnVtQbuq0bKa79TUEcPF3hjuX/k=", - "owner": "TECHNOFAB", - "repo": "nix-gitlab-ci", - "rev": "0c6949f585a2c1ea2cf85fc01445496f7c75faae", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "TECHNOFAB", - "repo": "nix-gitlab-ci", - "type": "gitlab" - } - }, - "root": { - "inputs": { - "nix-gitlab-ci-lib": "nix-gitlab-ci-lib" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/examples/rensa-nix/nix/repo/flake.nix b/examples/rensa-nix/nix/repo/flake.nix deleted file mode 100644 index ad3b77d..0000000 --- a/examples/rensa-nix/nix/repo/flake.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - inputs = { - nix-gitlab-ci-lib.url = "gitlab:TECHNOFAB/nix-gitlab-ci?dir=lib"; - }; - outputs = i: - i - // { - cilib = i.nix-gitlab-ci-lib.lib {inherit (i.parent) pkgs;}; - }; -} diff --git a/flake.lock b/flake.lock index 60df9c4..dd864f4 100644 --- a/flake.lock +++ b/flake.lock @@ -1,12 +1,227 @@ { "nodes": { + "cachix": { + "inputs": { + "devenv": [ + "devenv" + ], + "flake-compat": [ + "devenv" + ], + "git-hooks": [ + "devenv" + ], + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1728672398, + "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", + "owner": "cachix", + "repo": "cachix", + "rev": "aac51f698309fd0f381149214b7eee213c66ef0a", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "latest", + "repo": "cachix", + "type": "github" + } + }, + "devenv": { + "inputs": { + "cachix": "cachix", + "flake-compat": "flake-compat", + "git-hooks": [ + "git-hooks" + ], + "nix": "nix", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1732585607, + "narHash": "sha256-6ffeaSMuaL326f7KrCeScpSJtdHsFKS9gPrsSZkndvU=", + "owner": "cachix", + "repo": "devenv", + "rev": "a520f05c40ebecaf5e17064b27e28ba8e70c49fb", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "devenv", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1730504689, + "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "506278e768c2a08bec68eb62932193e341f55c90", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks": { + "inputs": { + "flake-compat": "flake-compat_2", + "gitignore": "gitignore", + "nixpkgs": "nixpkgs_4", + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1732021966, + "narHash": "sha256-mnTbjpdqF0luOkou8ZFi2asa1N3AA2CchR/RqCNmsGE=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "3308484d1a443fc5bc92012435d79e80458fe43c", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "type": "github" + }, + "original": { + "owner": "libgit2", + "repo": "libgit2", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": [ + "devenv" + ], + "flake-parts": "flake-parts", + "libgit2": "libgit2", + "nixpkgs": "nixpkgs_2", + "nixpkgs-23-11": [ + "devenv" + ], + "nixpkgs-regression": [ + "devenv" + ], + "pre-commit-hooks": [ + "devenv" + ] + }, + "locked": { + "lastModified": 1727438425, + "narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=", + "owner": "domenkozar", + "repo": "nix", + "rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.24", + "repo": "nix", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1764667669, - "narHash": "sha256-7WUCZfmqLAssbDqwg9cUDAXrSoXN79eEEq17qhTNM/Y=", + "lastModified": 1730531603, + "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "418468ac9527e799809c900eda37cbff999199b6", + "rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", "type": "github" }, "original": { @@ -18,43 +233,153 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1754184128, - "narHash": "sha256-AjhoyBL4eSyXf01Bmc6DiuaMrJRNdWopmdnMY0Pa/M0=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "02e72200e6d56494f4a7c0da8118760736e41b60", + "lastModified": 1730504152, + "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1730741070, + "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", "type": "github" }, "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", "type": "github" } }, - "ren": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, + "nixpkgs_2": { "locked": { - "dir": "lib", - "lastModified": 1758738378, - "narHash": "sha256-NjzqdvQCDDdObEBH8x/vdhbdhrIB+N9E570uCdksGHY=", - "owner": "rensa-nix", - "repo": "core", - "rev": "abe19f9f13aff41de2b63304545c87d193d19ef4", - "type": "gitlab" + "lastModified": 1717432640, + "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870", + "type": "github" }, "original": { - "dir": "lib", - "owner": "rensa-nix", - "repo": "core", - "type": "gitlab" + "owner": "NixOS", + "ref": "release-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1716977621, + "narHash": "sha256-Q1UQzYcMJH4RscmpTkjlgqQDX5yi1tZL0O345Ri6vXQ=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "4267e705586473d3e5c8d50299e71503f16a6fb6", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1730768919, + "narHash": "sha256-8AKquNnnSaJRXZxc5YmF/WfmxiHX6MMZZasRP6RRQkE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a04d33c0c3f1a59a2c1cb0c6e34cd24500e5a1dc", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1732238832, + "narHash": "sha256-sQxuJm8rHY20xq6Ah+GwIUkF95tWjGRd1X8xF+Pkk38=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8edf06bea5bcbee082df1b7369ff973b91618b8d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_6": { + "locked": { + "lastModified": 1731890469, + "narHash": "sha256-D1FNZ70NmQEwNxpSSdTXCSklBH1z2isPR84J6DQrJGs=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "5083ec887760adfe12af64830a66807423a859a7", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" } }, "root": { "inputs": { - "nixpkgs": "nixpkgs", - "ren": "ren" + "devenv": "devenv", + "flake-parts": "flake-parts_2", + "git-hooks": "git-hooks", + "nixpkgs": "nixpkgs_5", + "systems": "systems", + "treefmt-nix": "treefmt-nix" + } + }, + "systems": { + "locked": { + "lastModified": 1689347949, + "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", + "owner": "nix-systems", + "repo": "default-linux", + "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default-linux", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": "nixpkgs_6" + }, + "locked": { + "lastModified": 1732643199, + "narHash": "sha256-uI7TXEb231o8dkwB5AUCecx3AQtosRmL6hKgnckvjps=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "84637a7ab04179bdc42aa8fd0af1909fba76ad0c", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 769a6b9..596b693 100644 --- a/flake.nix +++ b/flake.nix @@ -1,37 +1,230 @@ { - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - ren.url = "gitlab:rensa-nix/core?dir=lib"; - }; - outputs = { - self, - ren, + flake-parts, + systems, ... } @ inputs: - ren.buildWith - { - inherit inputs; - cellsFrom = ./nix; - transformInputs = system: i: - i - // { - pkgs = import i.nixpkgs {inherit system;}; + flake-parts.lib.mkFlake {inherit inputs;} { + imports = [ + inputs.devenv.flakeModule + inputs.treefmt-nix.flakeModule + ./lib/flakeModule.nix + ]; + systems = import systems; + flake = {}; + perSystem = { + pkgs, + inputs', + config, + ... + }: rec { + treefmt = { + projectRootFile = "flake.nix"; + programs = { + alejandra.enable = true; + mdformat.enable = true; + yamlfmt.enable = true; + }; }; - cellBlocks = with ren.blocks; [ - (simple "devShells") - (simple "pkgs") - (simple "tests") - (simple "docs") - (simple "ci") - ]; - } - { - packages = ren.select self [ - ["repo" "tests"] - ["repo" "docs"] - ["repo" "ci" "packages"] - ["packages" "pkgs"] - ]; + devenv.shells.default = { + containers = pkgs.lib.mkForce {}; + packages = with pkgs; [dive skopeo]; + + pre-commit = { + hooks = { + treefmt = { + enable = true; + packageOverrides.treefmt = config.treefmt.build.wrapper; + }; + }; + }; + }; + ci = { + # use the image built in the parent pipeline for dogfooding + config.default-nix-image = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$CI_COMMIT_SHORT_SHA"; + stages = ["test"]; + jobs = { + "test" = { + stage = "test"; + nix = { + deps = [pkgs.hello pkgs.curl]; + disable-cache = false; + }; + variables = { + TEST = "test"; + TEST_WITH_DERIVATION = "${pkgs.hello}/test"; + }; + script = [ + "hello" + "curl google.de" + "echo $TEST $TEST_WITH_DERIVATION" + ]; + }; + "test-non-nix" = { + nix.enable = false; + stage = "test"; + image = "alpine:latest"; + script = [ + "echo \"This job will not be modified to use nix\"" + ]; + }; + }; + }; + + packages = let + setupScript = extra_setup: + pkgs.writeShellScriptBin "setup_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" + nix path-info --all > /tmp/nix-store-before + + if [ -z "$NIX_CI_DISABLE_CACHE" ]; then + ${extra_setup} + else + echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration" + fi + + export NIX_CONFIG=" + extra-trusted-public-keys = $NIX_PUBLIC_KEYS + extra-trusted-substituters = $NIX_SUBSTITUTERS + extra-substituters = $NIX_SUBSTITUTERS + $NIX_CONFIG + $NIX_EXTRA_CONFIG + " + echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" + + # load the job's deps only if the name was passed + if [[ ! -z $1 ]]; then + echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching deps for job" + nix build .#gitlab-ci-job-deps:$1 + source $(readlink -f result) + echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" + fi + ''; + finalizeScript = push_command: + pkgs.writeShellScriptBin "finalize_nix_ci" '' + echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing new store paths to cache" + nix path-info --all > /tmp/nix-store-after + ${pkgs.diffutils}/bin/diff --new-line-format="%L" \ + --old-line-format="" --unchanged-line-format="" \ + /tmp/nix-store-before /tmp/nix-store-after \ + | { + if [ -z "$NIX_CI_DISABLE_CACHE" ]; then + ${push_command} + else + ${pkgs.busybox}/bin/wc -l | { read count; echo "Caching disabled, not uploading $count new store entries..."; } + fi + } + echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" + ''; + mkImage = extraPackages: + pkgs.dockerTools.buildImage { + name = "nix-gitlab-ci"; + fromImage = pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-flakes"; + imageDigest = "sha256:d88e521662cb6bf9cef006b79ed6ed1069e297171f3c2585f2b898b30f7c045c"; + sha256 = "1pcbgxz9c98mfqrzyi14h568dw8vxj1kbgirnwl6vs8wfaamjaaf"; + finalImageName = "nixpkgs/nix-flakes"; + finalImageTag = "latest"; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = + [ + pkgs.gitMinimal + pkgs.gnugrep + ] + ++ extraPackages; + pathsToLink = ["/bin"]; + }; + }; + in { + setup-script = + setupScript + # sh + '' + # extra_setup + true + ''; + finalize-script = + finalizeScript + # sh + '' + # push_command + true + ''; + image = mkImage [ + (setupScript + # sh + '' + cachedir="$(pwd)/.nix-cache" + echo "Configuring caching with the Runner Cache in $cachedir..." + export NIX_SUBSTITUTERS="$NIX_SUBSTITUTERS file://$cachedir?priority=10&trusted=true" + '') + (finalizeScript + # sh + '' + # add ^* to all store paths ending in .drv (prevent warning log spam) + ${pkgs.gnused}/bin/sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "file://$(pwd)/.nix-cache" --stdin || true + '') + ]; + image-cachix = mkImage [ + (setupScript + # sh + '' + echo "Configuring caching with cachix..." + ${pkgs.cachix}/bin/cachix use $CACHIX_CACHE || true + '') + (finalizeScript + # sh + '' + ${pkgs.cachix}/bin/cachix push $CACHIX_CACHE || true + '') + ]; + image-attic = mkImage [ + (setupScript + # sh + '' + echo "Configuring caching with attic..." + ${pkgs.attic-client}/bin/attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true + ${pkgs.attic-client}/bin/attic use "$ATTIC_CACHE" || true + '') + (finalizeScript + # sh + '' + ${pkgs.attic-client}/bin/attic push --stdin ci:$ATTIC_CACHE || true + '') + ]; + }; + + checks = packages; + }; }; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + # flake & devenv related + flake-parts.url = "github:hercules-ci/flake-parts"; + systems.url = "github:nix-systems/default-linux"; + devenv = { + url = "github:cachix/devenv"; + inputs.git-hooks.follows = "git-hooks"; + }; + git-hooks.url = "github:cachix/git-hooks.nix"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + }; + + nixConfig = { + extra-substituters = [ + "https://cache.nixos.org/" + "https://nix-community.cachix.org" + "https://devenv.cachix.org" + ]; + + extra-trusted-public-keys = [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" + ]; + }; } diff --git a/gitlab-ci.yml b/gitlab-ci.yml new file mode 100644 index 0000000..f1cb469 --- /dev/null +++ b/gitlab-ci.yml @@ -0,0 +1,33 @@ +# +# NOTE: DEPRECATED: please switch to the CI/CD Component or include "templates/nix-gitlab-ci.yml" +# +variables: + # latest | latest-cachix | latest-attic etc. + NIX_CI_IMAGE_TAG: latest +stages: + - build + - trigger +nix-ci:build: + stage: build + image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:${NIX_CI_IMAGE_TAG} + before_script: + - source setup_nix_ci + script: + # build the generated-gitlab-ci.yml + - nix build .#gitlab-ci-config + - install result generated-gitlab-ci.yml + after_script: + # upload to binary cache + - finalize_nix_ci + artifacts: + paths: + - generated-gitlab-ci.yml +nix-ci:trigger: + stage: trigger + needs: + - nix-ci:build + trigger: + include: + - artifact: generated-gitlab-ci.yml + job: nix-ci:build + strategy: depend diff --git a/lib/VERSION b/lib/VERSION deleted file mode 100644 index ef538c2..0000000 --- a/lib/VERSION +++ /dev/null @@ -1 +0,0 @@ -3.1.2 diff --git a/lib/default.nix b/lib/default.nix deleted file mode 100644 index ff3ec88..0000000 --- a/lib/default.nix +++ /dev/null @@ -1,29 +0,0 @@ -args: let - # allow passing just pkgs aswell for convenience - lib = args.lib or args.pkgs.lib; - # makes it optional to pass if it's not explicitly needed - pkgs = args.pkgs or (throw "[nix-gitlab-ci] pkgs argument was used but not set, please pass it"); - inherit (lib) evalModules trimWith; - - impl = import ./impl {inherit lib pkgs cilib;}; - - cilib = { - inherit (impl) helpers modules mkPipeline mkJobRun mkJobDeps mkJobPatched; - utils = import ./utils.nix {inherit pkgs;}; - version = trimWith { - start = true; - end = true; - } (builtins.readFile ./VERSION); - - mkCI = config: - (evalModules { - modules = [ - cilib.modules.nixCiSubmodule - { - inherit config; - } - ]; - }).config; - }; -in - cilib diff --git a/lib/flake.nix b/lib/flake.nix index 227f9a6..5d00bb6 100644 --- a/lib/flake.nix +++ b/lib/flake.nix @@ -1,6 +1,9 @@ { - outputs = _i: { - lib = import ./.; - flakeModule = ./flakeModule.nix; - }; + description = "Nix-CI lib"; + + outputs = {...} @ inputs: + { + flakeModule = import ./flakeModule.nix; + } + // (import ./utils.nix); } diff --git a/lib/flakeModule.nix b/lib/flakeModule.nix index aadf292..64f7926 100644 --- a/lib/flakeModule.nix +++ b/lib/flakeModule.nix @@ -9,16 +9,260 @@ pkgs, ... }: let - cilib = import ./. {inherit lib pkgs;}; - inherit (lib) types mkOption; + cfg = config.ci.config; + + filterAttrsRec = pred: v: + if lib.isAttrs v + then lib.filterAttrs pred (lib.mapAttrs (path: filterAttrsRec pred) v) + else v; + + subType = options: lib.types.submodule {inherit options;}; + mkNullOption = type: + lib.mkOption { + default = null; + type = lib.types.nullOr type; + }; + + configType = with lib; + subType { + default-nix-image = mkOption { + type = types.str; + default = "registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:latest"; + description = "The image to use on nix jobs"; + }; + nix-jobs-per-default = mkOption { + type = types.bool; + default = true; + description = "Handle jobs nix-based by default or via opt-in (in a job set nix.enable = true) if false"; + }; + disable-cache = mkOption { + type = types.bool; + default = false; + description = "Whether to remove the cache key from all nix jobs and set NIX_CI_DISABLE_CACHE"; + }; + cache-key = mkOption { + type = types.str; + default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; + description = "Cache key to use for the nix cache"; + }; + }; + jobType = with lib; + subType { + # nix ci opts + nix = mkOption { + type = subType { + enable = mkOption { + type = types.bool; + default = cfg.nix-jobs-per-default; + description = "Handle this job as a nix job"; + }; + deps = mkOption { + type = types.listOf types.package; + default = []; + description = "Dependencies/packages to install for this job"; + }; + disable-cache = mkOption { + type = types.bool; + default = cfg.disable-cache; + description = "Whether to remove the cache key from this job and set NIX_CI_DISABLE_CACHE"; + }; + cache-key = mkOption { + type = types.str; + default = cfg.cache-key; + description = "Cache key to use for the nix cache"; + }; + }; + default = {}; + description = "Configure Nix Gitlab CI for each job individually"; + }; + # gitlab opts + script = mkOption { + type = types.listOf types.str; + default = []; + }; + stage = mkOption { + type = types.str; + default = "test"; + }; + image = mkOption { + type = types.str; + default = cfg.default-nix-image; + }; + after_script = mkNullOption (types.listOf types.str); + allow_failure = mkNullOption (types.either types.attrs types.bool); + artifacts = mkNullOption (types.attrs); + before_script = mkNullOption (types.listOf types.str); + cache = mkNullOption (types.either (types.listOf types.attrs) types.attrs); + coverage = mkNullOption (types.str); + dependencies = mkNullOption (types.listOf types.str); + environment = mkNullOption (types.either types.attrs types.str); + extends = mkNullOption (types.str); + hooks = mkNullOption (types.attrs); + id_tokens = mkNullOption (types.attrs); + "inherit" = mkNullOption (types.attrs); + interruptible = mkNullOption (types.bool); + needs = mkNullOption (types.listOf (types.either types.str types.attrs)); + publish = mkNullOption (types.str); + pages = mkNullOption (types.attrs); + parallel = mkNullOption (types.either types.int types.attrs); + release = mkNullOption (types.attrs); + retry = mkNullOption (types.either types.int types.attrs); + rules = mkNullOption (types.listOf types.attrs); + resource_group = mkNullOption (types.str); + secrets = mkNullOption (types.attrs); + services = mkNullOption (types.listOf types.attrs); + start_in = mkNullOption (types.str); + tags = mkNullOption (types.listOf types.str); + timeout = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + when = mkNullOption (types.str); + }; in { - options = { + options = with lib; { ci = mkOption { - type = types.submodule cilib.modules.nixCiSubmodule; + type = subType { + config = mkOption { + type = configType; + description = '' + Configuration options for the nix part itself + ''; + default = {}; + }; + image = mkNullOption (types.str); + variables = mkNullOption (types.attrs); + default = mkNullOption (types.attrs); + stages = mkNullOption (types.listOf types.str); + include = mkNullOption (types.attrs); + workflow = mkNullOption (types.attrs); + jobs = mkOption { + type = types.lazyAttrsOf jobType; + default = {}; + }; + }; + description = '' + Generate a Gitlab CI configuration which can be used to trigger a child pipeline. + This will inject code which pre-downloads the nix deps before each job and adds them to PATH. + ''; default = {}; }; }; - config.legacyPackages = config.ci.packages; + + config.legacyPackages = let + toYaml = (pkgs.formats.yaml {}).generate; + mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); + prepend = key: arr: job: + job + // lib.optionalAttrs job.nix.enable { + ${key} = + arr + ++ (job.${key} or []); + }; + append = key: arr: job: + job + // lib.optionalAttrs job.nix.enable { + ${key} = (job.${key} or []) ++ arr; + }; + prependToBeforeScript = prepend "before_script"; + appendToAfterScript = append "after_script"; + + # filter job's variables to either only those containing store paths + # or those that do not + filterJobVariables = nix: job: + lib.concatMapAttrs ( + name: value: + lib.optionalAttrs ((lib.hasInfix "/nix/store/" value) == nix) { + ${name} = value; + } + ) + (job.variables or {}); + + jobs = filterAttrsRec (n: v: v != null) config.ci.jobs; + rest = filterAttrsRec (n: v: v != null) (builtins.removeAttrs config.ci ["jobs" "config"]); + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + mapAttrs (key: job: let + variablesWithStorePaths = filterJobVariables true job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + in { + name = "gitlab-ci-job-deps:${key}"; + value = pkgs.writeShellScript "gitlab-ci-job-deps:${key}" '' + export PATH="${lib.makeBinPath job.nix.deps}:$PATH"; + ${variableExports} + ''; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + mapAttrs (key: job: let + variablesWithStorePaths = filterJobVariables false job; + variableExports = lib.concatLines ( + lib.mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths + ); + in { + name = "gitlab-ci-job:${key}"; + value = pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' + # set up deps and environment variables containing store paths + . ${jobsMappedForDeps."gitlab-ci-job-deps:${key}"} + # normal environment variables + ${variableExports} + # run before_script, script and after_script + echo -e "\e[32mRunning before_script...\e[0m" + ${lib.concatLines (job.before_script or [])} + echo -e "\e[32mRunning script...\e[0m" + ${lib.concatLines job.script} + echo -e "\e[32mRunning after_script...\e[0m" + ${lib.concatLines (job.after_script or [])} + ''; + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + mapAttrs (key: job: { + name = key; + value = assert lib.assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; + builtins.removeAttrs ( + (prependToBeforeScript [ + "source setup_nix_ci ${key}" + ] + (appendToAfterScript [ + "finalize_nix_ci" + ] + job)) + // lib.optionalAttrs job.nix.enable { + image = job.image; + variables = + (filterJobVariables false job) + // lib.optionalAttrs job.nix.disable-cache { + NIX_CI_DISABLE_CACHE = "yes"; + }; + cache = + ( + let + c = job.cache or []; + in + if builtins.isList c + then c + else [c] + ) + ++ (lib.optional (!job.nix.disable-cache) { + key = job.nix.cache-key; + paths = [".nix-cache/"]; + }); + } + ) ["nix"]; + }) + jobs; + in + { + gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript; } ); } diff --git a/lib/impl/default.nix b/lib/impl/default.nix deleted file mode 100644 index 7ff5623..0000000 --- a/lib/impl/default.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - pkgs, - lib, - cilib, -}: rec { - helpers = import ./helpers.nix {inherit lib pkgs;}; - mkJobDeps = import ./jobDeps.nix {inherit lib helpers;}; - mkJobRun = import ./jobRun.nix {inherit lib pkgs helpers;}; - mkJobPatched = import ./jobPatched.nix {inherit lib helpers;}; - mkPipeline = import ./pipeline.nix {inherit lib helpers mkJobDeps mkJobRun mkJobPatched;}; - modules = import ./modules {inherit lib cilib;}; -} diff --git a/lib/impl/helpers.nix b/lib/impl/helpers.nix deleted file mode 100644 index 754cead..0000000 --- a/lib/impl/helpers.nix +++ /dev/null @@ -1,123 +0,0 @@ -{ - pkgs, - lib, -} @ args: let - inherit - (lib) - types - isAttrs - filterAttrs - mapAttrs - mkOption - mkOptionType - isType - literalExpression - pipe - hasInfix - concatMapAttrs - optionalAttrs - ; -in rec { - prepend = key: arr: job: { - ${key} = arr ++ (job.${key} or []); - }; - append = key: arr: job: { - ${key} = (job.${key} or []) ++ arr; - }; - prependToBeforeScript = prepend "before_script"; - appendToAfterScript = append "after_script"; - - # json is also valid yaml and this removes dependency on jq and/or remarshal - # (used in pkgs.formats.json and pkgs.formats.yaml respectively) - toYaml = name: value: - pipe value [ - builtins.toJSON - builtins.unsafeDiscardOutputDependency - builtins.unsafeDiscardStringContext - (builtins.toFile name) - ]; - toYamlPretty = (pkgs.formats.yaml {}).generate; - - customMapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); - - filterAttrsRec = pred: v: - if isAttrs v - then filterAttrs pred (mapAttrs (_path: filterAttrsRec pred) v) - else v; - - # filter job's variables to either only those containing store paths - # or those that do not - filterJobVariables = shouldContain: job: - concatMapAttrs ( - name: value: - optionalAttrs ((hasInfix builtins.storeDir value) == shouldContain) { - ${name} = value; - } - ) - (job.variables or {}); - - deepMerge = lhs: rhs: - lhs - // rhs - // (builtins.mapAttrs ( - rName: rValue: let - lValue = lhs.${rName} or null; - in - if builtins.isAttrs lValue && builtins.isAttrs rValue - then deepMerge lValue rValue - else if builtins.isList lValue && builtins.isList rValue - then lValue ++ rValue - else rValue - ) - rhs); - - unsetType = mkOptionType { - name = "unset"; - description = "unset"; - descriptionClass = "noun"; - check = _value: true; - }; - unset = { - _type = "unset"; - }; - isUnset = isType "unset"; - unsetOr = typ: - (types.either unsetType typ) - // { - inherit (typ) description getSubOptions; - }; - mkUnsetOption = opts: - mkOption (opts - // { - type = unsetOr opts.type; - default = opts.default or unset; - defaultText = literalExpression "unset"; - }); - eitherWithSubOptions = typ_one: typ_two: - (types.either typ_one typ_two) - // { - getSubOptions = - if (typ_one.getSubOptions "test" != {}) - then typ_one.getSubOptions - else typ_two.getSubOptions; - }; - - filterUnset = value: - if builtins.isAttrs value && !builtins.hasAttr "_type" value - then let - filteredAttrs = builtins.mapAttrs (_n: filterUnset) value; - in - filterAttrs (_name: value: (!isUnset value)) filteredAttrs - else if builtins.isList value - then builtins.filter (elem: !isUnset elem) (map filterUnset value) - else value; - - # args.pkgs so "pkgs" does not need to be passed all the time - stdenvMinimal = args.pkgs.stdenvNoCC.override { - cc = null; - preHook = ""; - allowedRequisites = null; - initialPath = with args.pkgs; [coreutils findutils]; - extraNativeBuildInputs = []; - }; -} diff --git a/lib/impl/jobDeps.nix b/lib/impl/jobDeps.nix deleted file mode 100644 index 302597d..0000000 --- a/lib/impl/jobDeps.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ - lib, - helpers, -}: let - inherit (lib) concatLines mapAttrsToList makeBinPath; - inherit (helpers) filterJobVariables stdenvMinimal; -in - { - key, - job, - nixConfig, - }: let - variablesWithStorePaths = filterJobVariables true job; - variableExports = concatLines ( - mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithStorePaths - ); - script = '' - export PATH="${makeBinPath (nixConfig.deps or [])}:$PATH"; - # variables containing nix derivations: - ${variableExports} - ''; - in - stdenvMinimal.mkDerivation { - name = "gitlab-ci-job-deps-${key}"; - dontUnpack = true; - installPhase = - # sh - '' - echo '${script}' > $out - chmod +x $out - ''; - passthru = { - inherit script; - }; - } diff --git a/lib/impl/jobPatched.nix b/lib/impl/jobPatched.nix deleted file mode 100644 index 367a374..0000000 --- a/lib/impl/jobPatched.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - lib, - helpers, -}: let - inherit (lib) toList optionalAttrs optional; - inherit (helpers) prependToBeforeScript appendToAfterScript filterJobVariables; -in - { - key, - job, - pipelineName, - nixConfig, - }: - if ! nixConfig.enable - then job - else - (builtins.removeAttrs job ["variables" "cache"]) - // (prependToBeforeScript ["source setup_nix_ci \"gitlab-ci:pipeline:${pipelineName}:job-deps:${key}\""] job) - // (appendToAfterScript ["finalize_nix_ci"] job) - // (let - variables = - (filterJobVariables false job) - // optionalAttrs nixConfig.enableRunnerCache { - NIX_CI_CACHE_STRATEGY = "runner"; - }; - in - # filter empty variables - optionalAttrs (variables != {}) { - inherit variables; - }) - // (let - cache = - (toList (job.cache or [])) - ++ (optional nixConfig.enableRunnerCache { - key = nixConfig.runnerCacheKey; - paths = [".nix-cache/"]; - }); - in - # filter empty cache - optionalAttrs (cache != []) { - inherit cache; - }) diff --git a/lib/impl/jobRun.nix b/lib/impl/jobRun.nix deleted file mode 100644 index 4a1e0f6..0000000 --- a/lib/impl/jobRun.nix +++ /dev/null @@ -1,47 +0,0 @@ -{ - lib, - pkgs, - helpers, -}: let - inherit (lib) concatLines mapAttrsToList getExe; - inherit (helpers) filterJobVariables; -in - { - key, - job, - jobDeps, - }: let - variablesWithoutStorePaths = filterJobVariables false job; - variableExports = concatLines ( - mapAttrsToList (name: value: "export ${name}=\"${value}\"") variablesWithoutStorePaths - ); - sandboxHelper = pkgs.writeShellScriptBin "gitlab-ci-job-sandbox-helper" (builtins.readFile ./sandbox_helper.sh); - actualJobScript = pkgs.writeShellScript "gitlab-ci-job:${key}:raw" '' - # set up deps and environment variables containing store paths - . ${jobDeps} - # normal environment variables - ${variableExports} - # run before_script, script and after_script - echo -e "\e[32mRunning before_script...\e[0m" - set -x - ${concatLines (job.before_script or [])} - { set +x; } 2>/dev/null - echo -e "\e[32mRunning script...\e[0m" - set -x - ${concatLines job.script} - { set +x; } 2>/dev/null - echo -e "\e[32mRunning after_script...\e[0m" - set -x - ${concatLines (job.after_script or [])} - { set +x; } 2>/dev/null - ''; - in - # this way the sandbox helper just needs to be built once - pkgs.writeShellScriptBin "gitlab-ci-job:${key}" '' - exec ${getExe sandboxHelper} ${actualJobScript} $@ - '' - // { - passthru = { - inherit jobDeps actualJobScript; - }; - } diff --git a/lib/impl/modules/default.nix b/lib/impl/modules/default.nix deleted file mode 100644 index 9e33a7e..0000000 --- a/lib/impl/modules/default.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ - lib, - cilib, -}: rec { - inherit - (import ./root.nix { - inherit lib pipelineSubmodule soonixSubmodule; - }) - configSubmodule - nixCiSubmodule - ; - inherit - (import ./pipeline.nix { - inherit lib cilib jobSubmodule; - }) - pipelineConfigSubmodule - pipelineSubmodule - ; - inherit - (import ./job.nix { - inherit lib cilib; - }) - jobConfigSubmodule - jobSubmodule - ; - inherit - (import ./soonix.nix { - inherit lib cilib; - }) - soonixSubmodule - ; -} diff --git a/lib/impl/modules/job.nix b/lib/impl/modules/job.nix deleted file mode 100644 index fd4fc62..0000000 --- a/lib/impl/modules/job.nix +++ /dev/null @@ -1,663 +0,0 @@ -{ - lib, - cilib, - ... -}: let - inherit (lib) mkOption types filterAttrs; - inherit (cilib.helpers) filterUnset mkUnsetOption eitherWithSubOptions; -in rec { - jobConfigSubmodule = {pipelineConfig, ...}: { - options = { - enable = mkOption { - description = '' - Transform this job to a nix-configured one. - ''; - type = types.bool; - default = pipelineConfig.nixJobsByDefault; - }; - deps = mkOption { - description = '' - Dependencies to inject into the job before running it. - ''; - type = types.listOf types.package; - default = []; - }; - enableRunnerCache = mkOption { - type = types.bool; - default = false; - description = '' - Cache this job using the GitLab Runner cache. - - !!! warning - useful for tiny jobs, but most of the time it just takes an eternity. - ''; - }; - runnerCacheKey = mkOption { - type = types.str; - default = "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"; - description = '' - Cache key to use for the runner nix cache. Requires [`enableRunnerCache = true`](#pipelinesnamejobsnamenixenablerunnercache). - ''; - }; - }; - }; - - jobSubmodule = { - name, - config, - pipelineName, - pipelineConfig, - ... - }: let - # - # GITLAB OPTIONS - # - gitlabOptions = { - # see https://docs.gitlab.com/ci/yaml/ - after_script = mkUnsetOption { - type = types.listOf types.str; - description = '' - Override a set of commands that are executed after job. - - [Docs](https://docs.gitlab.com/ci/yaml/#after_script) - ''; - }; - allow_failure = mkUnsetOption { - type = eitherWithSubOptions types.bool (types.submodule { - options = { - exit_codes = mkUnsetOption { - type = types.either types.number (types.listOf types.number); - description = '' - Use `allow_failure.exit_codes` to control when a job should be allowed to fail. - The job is `allow_failure = true` for any of the listed exit codes, and `allow_failure = false` for any other exit code. - ''; - }; - }; - }); - description = '' - Allow job to fail. A failed job does not cause the pipeline to fail. - - [Docs](https://docs.gitlab.com/ci/yaml/#allow_failure) - ''; - }; - artifacts = mkUnsetOption { - # TODO: can be used in pipeline.default aswell - type = types.submodule { - options = { - paths = mkUnsetOption { - type = types.listOf types.str; - description = '' - Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can’t directly link outside it. - ''; - }; - excludes = mkUnsetOption { - type = types.listOf types.str; - description = '' - Use `exclude` to prevent files from being added to an artifacts archive. - ''; - }; - expire_in = mkUnsetOption { - type = types.str; - description = '' - Use `expire_in` to specify how long [job artifacts](https://docs.gitlab.com/ci/jobs/job_artifacts/) are stored before they expire and are deleted. - ''; - }; - expose_as = mkUnsetOption { - type = types.str; - description = '' - Use the `expose_as` keyword to [expose artifacts in the merge request UI](https://docs.gitlab.com/ci/jobs/job_artifacts/#link-to-job-artifacts-in-the-merge-request-ui). - ''; - }; - name = mkUnsetOption { - type = types.str; - description = '' - Use the `name` keyword to define the name of the created artifacts archive. You can specify a unique name for every archive. - ''; - }; - public = mkUnsetOption { - type = types.bool; - description = '' - Use `public` to determine whether the job artifacts should be publicly available. - ''; - }; - access = mkUnsetOption { - type = types.enum [ - "all" - "developer" - "maintainer" - "none" - ]; - description = '' - Use `access` to determine who can access the job artifacts from the GitLab UI or API. - This option does not prevent you from forwarding artifacts to downstream pipelines. - ''; - }; - reports = mkUnsetOption { - type = types.attrs; - description = '' - Use `reports` to collect artifacts generated by included templates in jobs. - ''; - }; - untracked = mkUnsetOption { - type = types.bool; - description = '' - Use `untracked` to add all Git untracked files as artifacts (along with the paths defined in `paths`). - `untracked` ignores configuration in the repository’s .gitignore, so matching artifacts in .gitignore are included. - ''; - }; - when = mkUnsetOption { - type = types.enum [ - "on_success" - "on_failure" - "always" - ]; - description = '' - Use `when` to upload artifacts on job failure or despite the failure. - ''; - }; - }; - }; - description = '' - List of files and directories to attach to a job on success. - - [Docs](https://docs.gitlab.com/ci/yaml/#artifacts) - ''; - }; - before_script = mkUnsetOption { - type = types.listOf types.str; - description = '' - Override a set of commands that are executed before job. - - [Docs](https://docs.gitlab.com/ci/yaml/#before_script) - ''; - }; - cache = mkUnsetOption { - # could be more granular - type = types.either (types.listOf types.attrs) types.attrs; - description = '' - List of files that should be cached between subsequent runs. - - [Docs](https://docs.gitlab.com/ci/yaml/#cache) - ''; - }; - coverage = mkUnsetOption { - type = types.str; - description = '' - Code coverage settings for a given job. - - [Docs](https://docs.gitlab.com/ci/yaml/#coverage) - ''; - }; - dast_configuration = mkUnsetOption { - type = types.attrs; - description = '' - Use configuration from DAST profiles on a job level. - - [Docs](https://docs.gitlab.com/ci/yaml/#dast_configuration) - ''; - }; - dependencies = mkUnsetOption { - type = types.listOf types.str; - description = '' - Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. - - [Docs](https://docs.gitlab.com/ci/yaml/#dependencies) - ''; - }; - environment = mkUnsetOption { - type = eitherWithSubOptions types.str (types.submodule { - options = { - name = mkUnsetOption { - type = types.str; - example = "production"; - description = '' - Set a name for an environment. - ''; - }; - url = mkUnsetOption { - type = types.str; - example = "https://prod.example.com"; - description = '' - Set a URL for an environment. - ''; - }; - on_stop = mkUnsetOption { - type = types.str; - example = "down"; - description = '' - Closing (stopping) environments can be achieved with the `on_stop` keyword defined under `environment`. - It declares a different job that runs to close the environment. - ''; - }; - action = mkUnsetOption { - type = types.enum ["start" "prepare" "stop" "verify" "access"]; - description = '' - Use the `action` keyword to specify how the job interacts with the environment. - ''; - }; - auto_stop_in = mkUnsetOption { - type = types.str; - description = '' - The `auto_stop_in` keyword specifies the lifetime of the environment. - When an environment expires, GitLab automatically stops it. - ''; - }; - kubernetes = mkUnsetOption { - type = types.attrs; # no we don't go that deep :D - description = '' - Use the `kubernetes` keyword to configure the - [dashboard for Kubernetes](https://docs.gitlab.com/ci/environments/kubernetes_dashboard/) and - [GitLab-managed Kubernetes resources](https://docs.gitlab.com/user/clusters/agent/managed_kubernetes_resources/) - for an environment. - ''; - }; - deployment_tier = mkUnsetOption { - type = types.enum [ - "production" - "staging" - "testing" - "development" - "other" - ]; - description = '' - Use the `deployment_tier` keyword to specify the tier of the deployment environment. - ''; - }; - }; - }); - description = '' - Name of an environment to which the job deploys. - See the implementation for nested options, or check out the docs: - - [Docs](https://docs.gitlab.com/ci/yaml/#environment) - ''; - example = { - name = "review/$CI_COMMIT_REF_SLUG"; - url = "https://$CI_COMMIT_REF_SLUG.review.example.com"; - action = "stop"; - auto_stop_in = "1 day"; - deployment_tier = "staging"; - }; - }; - extends = mkUnsetOption { - type = types.either types.str (types.listOf types.str); - description = '' - Configuration entries that this job inherits from. - - [Docs](https://docs.gitlab.com/ci/yaml/#extends) - ''; - }; - hooks = mkUnsetOption { - type = types.attrs; - description = '' - Use `hooks` to specify lists of commands to execute on the runner at certain stages of job execution, - like before retrieving the Git repository. - - [Docs](https://docs.gitlab.com/ci/yaml/#hooks) - ''; - }; - identity = mkUnsetOption { - type = types.str; - description = '' - Authenticate with third party services using identity federation. - - [Docs](https://docs.gitlab.com/ci/yaml/#identity) - ''; - }; - id_tokens = mkUnsetOption { - type = types.attrs; - description = '' - Use `id_tokens` to create [ID tokens](https://docs.gitlab.com/ci/secrets/id_token_authentication/) to authenticate with third party services - - [Docs](https://docs.gitlab.com/ci/yaml/#id_tokens) - ''; - example = { - ID_TOKEN_1.aud = "https://vault.example.com"; - ID_TOKEN_2.aud = [ - "https://gcp.com" - "https://aws.com" - ]; - SIGSTORE_ID_TOKEN.aud = "sigstore"; - }; - }; - image = mkOption { - # could be more granular - type = types.either types.str types.attrs; - description = '' - Container/OCI image to use for this job. - - !!! warning - Setting this will mess with Nix-GitLab-CI, so be careful and only use for non-nix jobs. - ''; - default = "$NIX_CI_IMAGE"; - example = { - name = "super/sql:experimental"; - entrypoint = [""]; - pull_policy = "if-not-present"; - docker = { - platform = "arm64/v8"; - user = "dave"; - }; - kubernetes.user = "1001"; - }; - }; - "inherit" = mkUnsetOption { - type = types.submodule { - options = { - default = mkUnsetOption { - type = types.either types.bool (types.listOf types.str); - description = '' - Use `inherit.default` to control the inheritance of [default keywords](https://docs.gitlab.com/ci/yaml/#default). - ''; - }; - variables = mkUnsetOption { - type = types.either types.bool (types.listOf types.str); - description = '' - Use `inherit.variables` to control the inheritance of [default variables](https://docs.gitlab.com/ci/yaml/#default-variables). - ''; - }; - }; - }; - description = '' - Select which global defaults all jobs inherit. - - [Docs](https://docs.gitlab.com/ci/yaml/#inherit) - ''; - }; - interruptible = mkUnsetOption { - type = types.bool; - description = '' - Defines if a job can be canceled when made redundant by a newer run. - - [Docs](https://docs.gitlab.com/ci/yaml/#interruptible) - ''; - }; - manual_confirmation = mkUnsetOption { - type = types.str; - description = '' - Define a custom confirmation message for a manual job. - - [Docs](https://docs.gitlab.com/ci/yaml/#manual_confirmation) - ''; - }; - needs = mkUnsetOption { - # could be done more granular - type = types.listOf (types.either types.str types.attrs); - description = '' - Execute jobs earlier than the stage ordering. - - [Docs](https://docs.gitlab.com/ci/yaml/#needs) - ''; - }; - pages = mkUnsetOption { - type = eitherWithSubOptions types.bool (types.submodule { - options = { - publish = mkUnsetOption { - type = types.str; - description = '' - Use `pages.publish` to configure the content directory of a [`pages` job](https://docs.gitlab.com/ci/yaml/#pages). - ''; - }; - path_prefix = mkUnsetOption { - type = types.str; - description = '' - Use `pages.path_prefix` to configure a path prefix for [parallel deployments](https://docs.gitlab.com/user/project/pages/#parallel-deployments) of GitLab Pages. - ''; - }; - expire_in = mkUnsetOption { - type = types.str; - description = '' - Use `expire_in` to specify how long a deployment should be available before it expires. - After the deployment is expired, it’s deactivated by a cron job running every 10 minutes. - ''; - }; - }; - }); - description = '' - Upload the result of a job to use with GitLab Pages. - - [Docs](https://docs.gitlab.com/ci/yaml/#pages) - ''; - }; - parallel = mkUnsetOption { - type = eitherWithSubOptions types.number (types.listOf (types.submodule { - options = { - matrix = mkUnsetOption { - type = types.attrs; - description = '' - Use `parallel.matrix` to run a job multiple times in parallel in a single pipeline, but with different variable values for each instance of the job. - ''; - }; - }; - })); - description = '' - How many instances of a job should be run in parallel. - - [Docs](https://docs.gitlab.com/ci/yaml/#parallel) - ''; - example = { - matrix = [ - { - PROVIDER = "aws"; - STACK = [ - "monitoring" - "app1" - "app2" - ]; - } - { - PROVIDER = "ovh"; - STACK = ["monitoring" "backup" "app"]; - } - { - PROVIDER = ["gcp" "vultr"]; - STACK = ["data" "processing"]; - } - ]; - }; - }; - release = mkUnsetOption { - # could be more granular - type = types.attrs; - description = '' - Instructs the runner to generate a [release](https://docs.gitlab.com/user/project/releases/) object. - - [Docs](https://docs.gitlab.com/ci/yaml/#release) - ''; - }; - resource_group = mkUnsetOption { - type = types.str; - description = '' - Limit job concurrency. - - [Docs](https://docs.gitlab.com/ci/yaml/#resource_group) - ''; - }; - retry = mkUnsetOption { - type = eitherWithSubOptions (types.ints.between 0 2) (types.submodule { - options = { - exit_codes = mkUnsetOption { - type = types.either types.int (types.listOf types.int); - description = '' - Use `retry.exit_codes` with `retry.max` to retry jobs for only specific failure cases. - ''; - }; - when = mkUnsetOption { - type = types.either types.str (types.listOf types.str); - description = '' - Use `retry.when` with `retry.max` to retry jobs for only specific failure cases. - ''; - }; - max = mkUnsetOption { - type = types.ints.between 0 2; - description = '' - `retry.max` is the maximum number of retries, like retry, and can be 0, 1, or 2. - ''; - }; - }; - }); - description = '' - When and how many times a job can be auto-retried in case of a failure. - - [Docs](https://docs.gitlab.com/ci/yaml/#retry) - ''; - }; - rules = mkUnsetOption { - # could be more granular - type = types.listOf types.attrs; - description = '' - List of conditions to evaluate and determine selected attributes of a job, and whether or not it’s created. - - [Docs](https://docs.gitlab.com/ci/yaml/#rules) - ''; - }; - script = mkOption { - type = types.listOf types.str; - description = '' - Shell script that is executed by a runner. - - [Docs](https://docs.gitlab.com/ci/yaml/#script) - ''; - }; - secrets = mkUnsetOption { - # could be more granular - type = types.attrs; - description = '' - The CI/CD secrets the job needs. - - [Docs](https://docs.gitlab.com/ci/yaml/#secrets) - ''; - }; - services = mkUnsetOption { - # could be more granular - type = types.attrs; - description = '' - Use Docker services images. - - [Docs](https://docs.gitlab.com/ci/yaml/#services) - ''; - }; - stage = mkOption { - type = types.str; - description = '' - Defines a job stage. - - [Docs](https://docs.gitlab.com/ci/yaml/#stage) - ''; - }; - tags = mkUnsetOption { - type = types.listOf types.str; - description = '' - List of tags that are used to select a runner. - - [Docs](https://docs.gitlab.com/ci/yaml/#tags) - ''; - }; - timeout = mkUnsetOption { - type = types.str; - description = '' - Define a custom job-level timeout that takes precedence over the project-wide setting. - - [Docs](https://docs.gitlab.com/ci/yaml/#timeout) - ''; - }; - trigger = mkUnsetOption { - # could be more granular - type = types.either types.str types.attrs; - description = '' - Defines a downstream pipeline trigger. - - [Docs](https://docs.gitlab.com/ci/yaml/#trigger) - ''; - }; - when = mkUnsetOption { - type = types.enum ["on_success" "on_failure" "never" "always" "manual" "delayed"]; - description = '' - When to run job. See also [`manual_confirmation`](#pipelinesnamejobsnamemanual_confirmation) - - [Docs](https://docs.gitlab.com/ci/yaml/#when) - ''; - }; - variables = mkUnsetOption { - type = types.attrsOf types.str; - description = '' - You can use job variables in commands in the job’s `script`, `before_script`, or `after_script` sections, and also with some job keywords. - Check the **Supported values** section of each job keyword to see if it supports variables. - - [Docs](https://docs.gitlab.com/ci/yaml/#job-variables) - ''; - }; - }; - in { - options = - { - nix = mkOption { - description = '' - Nix-GitLab-CI config options for this job. - ''; - type = types.submoduleWith { - modules = [jobConfigSubmodule]; - specialArgs.pipelineConfig = pipelineConfig; - }; - default = {}; - }; - finalConfig = mkOption { - description = '' - Final configuration of this job. (readonly) - ''; - readOnly = true; - internal = true; - type = types.attrs; - }; - depsDrv = mkOption { - description = '' - Derivation containing all the dependencies of this job. (readonly) - ''; - readOnly = true; - internal = true; - type = types.package; - }; - runnerDrv = mkOption { - description = '' - Derivation containing the script for running the job locally. (readonly) - ''; - readOnly = true; - internal = true; - type = types.package; - }; - packages = mkOption { - description = '' - Final packages for this job, eg. for running the job or getting it's deps. (readonly) - ''; - readOnly = true; - internal = true; - type = types.attrsOf types.package; - }; - } - // gitlabOptions; - config = let - attrsToKeep = builtins.attrNames gitlabOptions; - job = filterUnset (filterAttrs (n: _v: builtins.elem n attrsToKeep) config); - in { - finalConfig = cilib.mkJobPatched { - key = name; - nixConfig = config.nix; - inherit job pipelineName; - }; - depsDrv = cilib.mkJobDeps { - key = name; - nixConfig = config.nix; - inherit job; - }; - runnerDrv = cilib.mkJobRun { - key = name; - jobDeps = config.depsDrv; - inherit job; - }; - packages = { - "gitlab-ci:pipeline:${pipelineName}:job-deps:${name}" = config.depsDrv; - "gitlab-ci:pipeline:${pipelineName}:job:${name}" = config.runnerDrv; - }; - }; - }; -} diff --git a/lib/impl/modules/pipeline.nix b/lib/impl/modules/pipeline.nix deleted file mode 100644 index 32b9e0c..0000000 --- a/lib/impl/modules/pipeline.nix +++ /dev/null @@ -1,228 +0,0 @@ -{ - lib, - cilib, - jobSubmodule, - ... -}: let - inherit (lib) mkOption types filterAttrs mergeAttrsList mapAttrs mkRenamedOptionModule literalExpression; - inherit (cilib.helpers) filterUnset unset mkUnsetOption toYaml toYamlPretty eitherWithSubOptions; - - pipelineConfigSubmodule = {rootConfig, ...}: { - options = { - nixJobsByDefault = mkOption { - description = '' - Whether to transform all jobs to nix-configured jobs by default. - If false, you need to set `nix.enable` for each job you want to be transformed. - ''; - type = types.bool; - default = rootConfig.nixJobsByDefault; - }; - }; - }; - - pipelineSubmodule = { - name, - config, - rootConfig, - ... - }: let - # - # GITLAB OPTIONS - # - gitlabOptions = { - default = mkOption { - type = types.submodule { - # allow other keys aswell, maybe define them in the future? https://docs.gitlab.com/ci/yaml/#default - freeformType = types.anything; - options = { - image = mkUnsetOption { - type = types.str; - description = '' - Default image to use for this entire pipeline. - - !!! note - Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). - ''; - }; - services = mkUnsetOption { - type = types.listOf types.anything; - description = '' - !!! note - Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). - ''; - }; - cache = mkUnsetOption { - # could be more granular - type = types.either (types.listOf types.attrs) types.attrs; - description = '' - !!! note - Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). - ''; - }; - before_script = mkUnsetOption { - type = types.listOf types.str; - description = '' - !!! note - Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). - ''; - }; - after_script = mkUnsetOption { - type = types.listOf types.str; - description = '' - !!! note - Moved from top level to `default`: [GitLab Docs](https://docs.gitlab.com/ci/yaml/deprecated_keywords/). - ''; - }; - }; - }; - # required for it to show up in the docs, but not in the config - default = unset; - defaultText = literalExpression "unset"; - description = '' - Custom default values for job keywords. - ''; - }; - include = mkUnsetOption { - type = types.attrs; - description = '' - Import configuration from other YAML files. - - [Docs](https://docs.gitlab.com/ci/yaml/#include) - ''; - }; - stages = mkOption { - type = types.listOf types.str; - default = []; - # .pre and .post always exist - apply = val: [".pre"] ++ val ++ [".post"]; - description = '' - The names and order of the pipeline stages. - - [Docs](https://docs.gitlab.com/ci/yaml/#stages) - - !!! note - `.pre` and `.post` are added automatically. - ''; - }; - variables = mkUnsetOption { - # default/global variables can have descriptions etc. - type = types.attrsOf (eitherWithSubOptions types.str (types.submodule { - options = { - description = mkUnsetOption { - type = types.str; - description = '' - Use the `description` keyword to define a description for a default variable. - ''; - }; - value = mkUnsetOption { - type = types.str; - description = '' - Use the `value` keyword to define a pipeline-level (default) variable’s value. - ''; - }; - options = mkUnsetOption { - type = types.listOf types.str; - description = '' - Use `options` to define an array of values that are [selectable in the UI when running a pipeline manually](https://docs.gitlab.com/ci/pipelines/#configure-a-list-of-selectable-prefilled-variable-values). - ''; - }; - expand = mkUnsetOption { - type = types.bool; - description = '' - Use the `expand` keyword to configure a variable to be expandable or not. - ''; - }; - }; - })); - description = '' - Define default CI/CD variables for all jobs in the pipeline. - Supports strings or attrs as values, for more info see [here](https://docs.gitlab.com/ci/yaml/#variablesdescription). - - [Docs](https://docs.gitlab.com/ci/yaml/#default-variables) - ''; - }; - workflow = mkUnsetOption { - type = types.attrs; - description = '' - Control what types of pipeline run. - - [Docs](https://docs.gitlab.com/ci/yaml/#workflow) - ''; - }; - }; - in { - _file = ./pipeline.nix; - # from https://docs.gitlab.com/ci/yaml/#default - imports = builtins.map (val: mkRenamedOptionModule [val] ["default" val]) [ - "after_script" - "artifacts" - "before_script" - "cache" - "hooks" - "id_tokens" - "image" - "interruptible" - "retry" - "services" - "tags" - "timeout" - ]; - options = - { - nix = mkOption { - description = '' - Nix-GitLab-CI config options for this pipeline. - ''; - type = types.submoduleWith { - modules = [pipelineConfigSubmodule]; - specialArgs.rootConfig = rootConfig; - }; - default = {}; - }; - finalConfig = mkOption { - description = '' - Final config of the pipeline. (readonly) - ''; - readOnly = true; - type = types.attrs; - }; - packages = mkOption { - description = '' - Final packages for use in CI. (readonly) - ''; - readOnly = true; - type = types.attrsOf types.package; - }; - # jobs are nested to make distinguishing them from other keys in the ci config easier - jobs = mkOption { - description = '' - Jobs for this pipeline. - ''; - type = types.attrsOf (types.submoduleWith { - modules = [jobSubmodule]; - specialArgs = { - pipelineName = name; - pipelineConfig = config.nix; - }; - }); - default = {}; - }; - } - // gitlabOptions; - config = let - attrsToKeep = builtins.attrNames gitlabOptions; - in { - finalConfig = - (filterUnset (filterAttrs (n: _v: builtins.elem n attrsToKeep) config)) - // mapAttrs (_name: value: value.finalConfig) config.jobs; - packages = - { - "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-config.json" config.finalConfig; - "gitlab-ci:pipeline:${name}:pretty" = toYamlPretty "gitlab-ci-config.yml" config.finalConfig; - } - // mergeAttrsList (map (job: job.packages) (builtins.attrValues config.jobs)); - }; - }; -in { - inherit pipelineSubmodule pipelineConfigSubmodule; -} diff --git a/lib/impl/modules/root.nix b/lib/impl/modules/root.nix deleted file mode 100644 index bd99050..0000000 --- a/lib/impl/modules/root.nix +++ /dev/null @@ -1,74 +0,0 @@ -{ - lib, - soonixSubmodule, - pipelineSubmodule, - ... -}: let - inherit (lib) mkOption types foldr; -in rec { - configSubmodule = { - options = { - soonix = mkOption { - description = '' - Configure the soonix `.gitlab-ci.yml` generation. - ''; - type = types.submodule soonixSubmodule; - default = {}; - }; - nixJobsByDefault = mkOption { - description = '' - Whether to transform all jobs to nix-configured jobs by default. - If false, you need to set [`nix.enable`](#pipelinesnamejobsnamenixenable) - for each job you want to be transformed. - ''; - type = types.bool; - default = true; - }; - }; - }; - - nixCiSubmodule = {config, ...}: { - options = { - config = mkOption { - description = '' - Configuration of Nix-GitLab-CI itself. - ''; - type = types.submodule configSubmodule; - default = {}; - }; - pipelines = mkOption { - description = '' - Defines all pipelines. - ''; - type = types.attrsOf (types.submoduleWith { - modules = [pipelineSubmodule]; - specialArgs.rootConfig = config.config; - }); - default = {}; - }; - - packages = mkOption { - description = '' - Final packages for use in CI. (readonly) - ''; - readOnly = true; - type = types.attrsOf types.package; - }; - soonix = mkOption { - description = '' - Soonix config for generating `.gitlab-ci.yml`. (readonly) - - See [`config.soonix`](#configsoonix) for configuring this. - ''; - readOnly = true; - type = types.attrs; - }; - }; - config = { - packages = foldr (pipeline: acc: acc // pipeline) {} ( - map (pipeline: pipeline.packages) (builtins.attrValues config.pipelines) - ); - soonix = config.config.soonix.finalConfig; - }; - }; -} diff --git a/lib/impl/modules/soonix.nix b/lib/impl/modules/soonix.nix deleted file mode 100644 index d23d727..0000000 --- a/lib/impl/modules/soonix.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - lib, - cilib, - ... -}: let - inherit (lib) mkOption types; -in { - soonixSubmodule = {config, ...}: { - options = { - componentVersion = mkOption { - description = '' - CI/CD component version. Also get's passed to inputs → version. - ''; - type = types.str; - default = cilib.version; - }; - componentUrl = mkOption { - description = '' - CI/CD component url. - ''; - type = types.str; - default = "gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci"; - }; - componentInputs = mkOption { - description = '' - Extra inputs to pass to the CI/CD component. - ''; - type = types.attrs; - default = {}; - }; - extraData = mkOption { - description = '' - Extra data to include in the `.gitlab-ci.yml` file. - ''; - type = types.attrs; - default = {}; - }; - - finalConfig = mkOption { - internal = true; - type = types.attrs; - }; - }; - config.finalConfig = { - opts.format = "yaml"; - hook = { - mode = "copy"; - gitignore = false; - }; - output = ".gitlab-ci.yml"; - generator = "nix"; - data = - cilib.helpers.deepMerge - { - include = [ - { - component = "${config.componentUrl}@${config.componentVersion}"; - inputs = - { - version = config.componentVersion; - } - // config.componentInputs; - } - ]; - } - config.extraData; - }; - }; -} diff --git a/lib/impl/pipeline.nix b/lib/impl/pipeline.nix deleted file mode 100644 index 4d9b185..0000000 --- a/lib/impl/pipeline.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ - lib, - helpers, - mkJobDeps, - mkJobRun, - mkJobPatched, -}: let - inherit (lib) assertMsg; - inherit (helpers) filterAttrsRec customMapAttrs toYaml toYamlPretty; -in - { - name, - pipeline, - nixConfig, - }: let - jobs = filterAttrsRec (_n: v: v != null) pipeline.jobs; - rest = filterAttrsRec (_n: v: v != null) (builtins.removeAttrs pipeline ["jobs"]); - # this allows us to nix build this to get all the mentioned dependencies from the binary cache - # pro: we don't have to download everything, just the deps for the current job - # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file - # we can support different architectures between runners (eg. the arch of the initial runner does not matter) - jobsMappedForDeps = - customMapAttrs (key: job: { - name = "gitlab-ci:pipeline:${name}:job-deps:${key}"; - value = mkJobDeps {inherit key job nixConfig;}; - }) - jobs; - # allows the user to directly run the script - jobsMappedForScript = - customMapAttrs (key: job: { - name = "gitlab-ci:pipeline:${name}:job:${key}"; - value = mkJobRun { - inherit key job nixConfig; - jobDeps = jobsMappedForDeps."gitlab-ci:pipeline:${name}:job-deps:${key}"; - }; - }) - jobs; - # build the deps specific for this job before anything, this way the deps should be fetched from the cache - jobsPatched = - customMapAttrs (key: job: { - name = key; - value = assert assertMsg (builtins.elem job.stage (rest.stages or [])) "stage '${job.stage}' of job '${key}' does not exist"; - mkJobPatched { - inherit key job nixConfig; - pipelineName = name; - }; - }) - jobs; - in { - packages = - # gitlab-ci:pipeline: - # gitlab-ci:pipeline::job: - # gitlab-ci:pipeline::job-deps: - { - "gitlab-ci:pipeline:${name}" = toYaml "gitlab-ci-${name}.yml" (rest // jobsPatched); - "gitlab-ci:pipeline:${name}:pretty" = toYamlPretty "gitlab-ci-${name}.yml" (rest // jobsPatched); - } - // jobsMappedForDeps - // jobsMappedForScript; - finalConfig = rest // jobsPatched; - } diff --git a/lib/impl/sandbox_helper.sh b/lib/impl/sandbox_helper.sh deleted file mode 100644 index 8b76ae1..0000000 --- a/lib/impl/sandbox_helper.sh +++ /dev/null @@ -1,75 +0,0 @@ -echo -e "\e[32mSetting up...\e[0m" - -actualJobScript=$1 -shift - -INCLUDE_DIRTY=false -NO_SANDBOX=false -KEEP_TMP=false -KEEP_ENV="" -# parse flags -while [[ $# -gt 0 ]]; do - case "$1" in - --include-dirty) - INCLUDE_DIRTY=true - shift - ;; - --no-sandbox) - NO_SANDBOX=true - shift - ;; - --keep-tmp) - KEEP_TMP=true - shift - ;; - --keep-env) - KEEP_ENV="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - echo "use --include-dirty, --no-sandbox, --keep-tmp and --keep-env " >&2 - exit 1 - ;; - esac -done - -if [ $NO_SANDBOX = false ]; then - echo "Running with simple sandboxing" - NGCI_TMPDIR=$(mktemp -dt "nix-gitlab-ci.XXX") - if [ $KEEP_TMP = false ]; then - trap "rm -rf '$NGCI_TMPDIR'" EXIT - else - echo "Temp dir will be preserved at: $NGCI_TMPDIR" - fi - - # check if dirty - DIRTY_PATCH="" - if ! git diff --quiet && ! git diff --staged --quiet; then - echo "Warning: working tree is dirty." - DIRTY_PATCH=$(mktemp -t "nix-gitlab-ci.XXX.patch") - git diff --staged > "$DIRTY_PATCH" - trap "rm -f '$DIRTY_PATCH'" EXIT - fi - git clone . $NGCI_TMPDIR - pushd $NGCI_TMPDIR >/dev/null - if [[ ! -z "$DIRTY_PATCH" && $INCLUDE_DIRTY = true ]]; then - echo "Copying dirty changes..." - git apply "$DIRTY_PATCH" 2>/dev/null || echo "Failed to copy dirty changes" - git add . # required so the files are staged again - fi - - echo "Running job in $NGCI_TMPDIR" - env -i $( - if [[ -n "$KEEP_ENV" ]]; then - IFS=',' read -ra VARS <<< "$KEEP_ENV" - for var in "${VARS[@]}"; do - printf '%s=%q ' "$var" "${!var}" - done - fi - ) bash $actualJobScript - popd >/dev/null -else - exec $actualJobScript -fi - diff --git a/lib/utils.nix b/lib/utils.nix index ae15284..a40dadd 100644 --- a/lib/utils.nix +++ b/lib/utils.nix @@ -1,46 +1,46 @@ -{pkgs, ...}: { - commitAndPushFiles = { - message, - files ? [], - }: jobArgs: - jobArgs - // { - before_script = - (jobArgs.before_script or []) - ++ [ - # sh - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" - eval "$(ssh-agent -s)" >/dev/null; - mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; - ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; - echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; - git config --global user.email "$GIT_EMAIL" >/dev/null; - git config --global user.name "$GIT_NAME" >/dev/null; - export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; - git remote rm origin && git remote add origin ''${CI_PUSH_REPO} - echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" - '' - ]; - script = let - addScript = - if builtins.length files == 0 - then "" - else "git add ${builtins.concatStringsSep " " files}"; - in - (jobArgs.script or []) - ++ [ - # sh - '' - echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" - ${addScript} - git diff --cached --exit-code >/dev/null && - echo "Nothing to commit" || - git commit -m "${message}" --no-verify; - git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip - echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" - '' - ]; - nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; - }; +{ + mkUtils = {pkgs, ...}: { + commitAndPushFiles = { + message, + files ? [], + }: jobArgs: + jobArgs + // { + before_script = + (jobArgs.before_script or []) + ++ [ + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit_setup[collapsed=true]\\r\\e[0KSetting up commitAndPushFiles" + eval "$(ssh-agent -s)" >/dev/null; + mkdir -p ~/.ssh; touch ~/.ssh/known_hosts; + ssh-keyscan -t rsa $CI_SERVER_HOST >> ~/.ssh/known_hosts; + echo "$GIT_SSH_PRIV_KEY" | tr -d '\r' | ssh-add - >/dev/null; + git config --global user.email "$GIT_EMAIL" >/dev/null; + git config --global user.name "$GIT_NAME" >/dev/null; + export CI_PUSH_REPO=`echo $CI_REPOSITORY_URL | sed -e "s|.*@\(.*\)|git@\1|" -e "s|/|:|"`; + git remote rm origin && git remote add origin ''${CI_PUSH_REPO} + echo -e "\\e[0Ksection_end:`date +%s`:commit_setup\\r\\e[0K" + '' + ]; + script = let + addScript = + if builtins.length files == 0 + then "" + else "git add ${builtins.concatStringsSep " " files}"; + in + (jobArgs.script or []) + ++ [ + '' + echo -e "\\e[0Ksection_start:`date +%s`:commit[collapsed=true]\\r\\e[0KCommiting & pushing changes if necessary" + ${addScript} + git diff --cached --exit-code >/dev/null && + echo "Nothing to commit" || + git commit -m "${message}" --no-verify; + git push --tags origin ''${GIT_SOURCE_REF:-HEAD}:''${GIT_TARGET_REF:-$CI_COMMIT_REF_NAME} -o ci.skip + echo -e "\\e[0Ksection_end:`date +%s`:commit\\r\\e[0K" + '' + ]; + nix.deps = (jobArgs.nix.deps or []) ++ [pkgs.openssh pkgs.gitMinimal pkgs.gnused]; + }; + }; } diff --git a/nix/packages/pkgs.nix b/nix/packages/pkgs.nix deleted file mode 100644 index 34d07aa..0000000 --- a/nix/packages/pkgs.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - inputs, - system, - ... -}: let - inherit (inputs) pkgs; -in rec { - setupScript = pkgs.writeShellScriptBin "setup_nix_ci" (builtins.readFile ./scripts/setup_nix_ci.sh); - finalizeScript = pkgs.writeShellScriptBin "finalize_nix_ci" (builtins.readFile ./scripts/finalize_nix_ci.sh); - image = pkgs.dockerTools.buildImage { - name = "nix-ci"; - fromImage = let - hashes = { - "x86_64-linux" = "sha256-kJ7dqje5o1KPr3RDZ7/THbhMSoiCU1C/7HshDrNfwnM="; - "aarch64-linux" = "sha256-jz+Z3Ji+hy5d9ImOh/YOKCqy9P9/cseSov+5J/O95bg="; - }; - # check digest of tags like nixos-24.11-aarch64-linux etc. - digests = { - "x86_64-linux" = "sha256:345f210dea4cbd049e2d01d13159c829066dfb6e273cdd49ea878186d17b19f7"; - "aarch64-linux" = "sha256:66163fdf446d851416dd4e9be28c0794d9c2550214a57a846957699a3f5747f6"; - }; - hash = hashes.${system} or (throw "Unsupported system"); - imageDigest = digests.${system} or (throw "Unsupported system"); - in - pkgs.dockerTools.pullImage { - imageName = "nixpkgs/nix-flakes"; - inherit hash imageDigest; - }; - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = with pkgs; - [ - gitMinimal - gnugrep - gnused - coreutils - diffutils - cachix - attic-client - ] - ++ [ - setupScript - finalizeScript - ]; - pathsToLink = ["/bin"]; - }; - }; -} diff --git a/nix/packages/scripts/finalize_nix_ci.sh b/nix/packages/scripts/finalize_nix_ci.sh deleted file mode 100644 index ecc2211..0000000 --- a/nix/packages/scripts/finalize_nix_ci.sh +++ /dev/null @@ -1,40 +0,0 @@ -echo -e "\\e[0Ksection_start:`date +%s`:finalize_nix_ci[collapsed=true]\\r\\e[0KFinalizing Nix CI..." - nix path-info --all > /tmp/nix-store-after - echo "Finding new paths..." - NEW_PATHS=$(diff --new-line-format="%L" \ - --old-line-format="" --unchanged-line-format="" \ - /tmp/nix-store-before /tmp/nix-store-after) - COUNT=$(wc -l <<<"$NEW_PATHS") - - if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then - export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}"; - fi - - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - echo -e "\\e[0Ksection_start:`date +%s`:cache_push[collapsed=true]\\r\\e[0KPushing $COUNT new store paths to cache ($NIX_CI_CACHE_STRATEGY)" - echo -n "$NEW_PATHS" | { - case "$NIX_CI_CACHE_STRATEGY" in - "runner") - export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} - # add ^* to all store paths ending in .drv (prevent warning log spam) - sed '/\.drv$/s/$/^*/' | nix copy --quiet --to "$RUNNER_CACHE" --stdin || true - ;; - "attic") - attic push --stdin ci:$ATTIC_CACHE || true - ;; - "cachix") - cachix push $CACHIX_CACHE || true - ;; - "none") - echo "Cache strategy is none, doing nothing..." - ;; - *) - echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'" - ;; - esac - } - echo -e "\\e[0Ksection_end:`date +%s`:cache_push\\r\\e[0K" - else - echo "Caching disabled, not uploading $COUNT new store entries..." - fi -echo -e "\\e[0Ksection_end:`date +%s`:finalize_nix_ci\\r\\e[0K" diff --git a/nix/packages/scripts/setup_nix_ci.sh b/nix/packages/scripts/setup_nix_ci.sh deleted file mode 100644 index eaf189b..0000000 --- a/nix/packages/scripts/setup_nix_ci.sh +++ /dev/null @@ -1,48 +0,0 @@ -echo -e "\\e[0Ksection_start:`date +%s`:nix_setup[collapsed=true]\\r\\e[0KSetting up Nix CI" - nix path-info --all > /tmp/nix-store-before - - if [[ "$NIX_CI_CACHE_STRATEGY" == "auto" ]]; then - export NIX_CI_CACHE_STRATEGY="${NIX_CI_RUNNER_CACHE_STRATEGY:-${NIX_CI_DEFAULT_CACHE_STRATEGY:-none}}"; - echo "NIX_CI_CACHE_STRATEGY was set to auto, selected '$NIX_CI_CACHE_STRATEGY' for this job" - fi - - if [ -z "$NIX_CI_DISABLE_CACHE" ]; then - echo -e "\\e[0Ksection_start:`date +%s`:cache_setup[collapsed=true]\\r\\e[0KConfiguring cache ($NIX_CI_CACHE_STRATEGY)" - case "$NIX_CI_CACHE_STRATEGY" in - "runner") - export RUNNER_CACHE=''${RUNNER_CACHE:-"file://$(pwd)/.nix-cache"} - echo "Runner Cache: $RUNNER_CACHE" - export NIX_CONFIG="$NIX_CONFIG - extra-trusted-substituters = $RUNNER_CACHE?priority=10&trusted=true - extra-substituters = $RUNNER_CACHE?priority=10&trusted=true - " - ;; - "attic") - echo "Attic Cache: $ATTIC_CACHE" - attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" || true - attic use "$ATTIC_CACHE" || true - ;; - "cachix") - echo "Cachix Cache: $CACHIX_CACHE" - cachix use "$CACHIX_CACHE" || true - ;; - "none") - echo "Cache strategy is none, doing nothing..." - ;; - *) - echo "WARNING: Invalid cache strategy set: '$NIX_CI_CACHE_STRATEGY'" - ;; - esac - echo -e "\\e[0Ksection_end:`date +%s`:cache_setup\\r\\e[0K" - else - echo "Caching disabled (NIX_CI_DISABLE_CACHE), skipping cache configuration..." - fi - - # load the job's deps only if the name was passed - if [[ ! -z $1 ]]; then - echo -e "\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KFetching Nix dependencies for job" - nix build .#$1 - source $(readlink -f result) - echo -e "\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K" - fi -echo -e "\\e[0Ksection_end:`date +%s`:nix_setup\\r\\e[0K" diff --git a/nix/repo/ci.nix b/nix/repo/ci.nix deleted file mode 100644 index 157ede7..0000000 --- a/nix/repo/ci.nix +++ /dev/null @@ -1,100 +0,0 @@ -{inputs, ...}: let - inherit (inputs) cilib; -in - cilib.mkCI { - config.soonix = { - componentUrl = "$CI_SERVER_FQDN/$CI_PROJECT_PATH/nix-gitlab-ci"; - componentVersion = "$CI_COMMIT_SHORT_SHA"; - componentInputs.cache_files = ["flake.*" "nix/repo/ci.nix"]; - # bootstrapping still needs to be done in the gitlab-ci.yml directly, - # the child pipeline can then use the built images to test them - extraData = { - stages = ["build-images" "build" "trigger"]; - variables.NIX_CI_IMAGE = "$CI_REGISTRY_IMAGE/nix-ci:$CI_COMMIT_SHORT_SHA"; - "build:image" = { - stage = "build-images"; - parallel.matrix = [ - {ARCH = ["x86_64-linux" "aarch64-linux"];} - ]; - image = "nixpkgs/nix-flakes:latest"; - script = ["nix build .#image --system $ARCH"]; - after_script = ["install -D result dist/nix-ci-$ARCH.tar.gz"]; - artifacts.paths = ["dist"]; - }; - "deploy:image" = { - stage = "build-images"; - image = "nixpkgs/nix-flakes:latest"; - needs = ["build:image"]; - before_script = [ - # sh - '' - nix profile install nixpkgs#buildah - export PATH="$PATH:$HOME/.nix-profile/bin" - export REGISTRY_AUTH_FILE=''${HOME}/auth.json - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - mkdir -p /etc/containers && echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json - mkdir -p /var/tmp - '' - ]; - script = [ - # sh - '' - export NORMALIZED_BRANCH=''${CI_COMMIT_BRANCH/\//-} - buildah manifest create localhost/nix-ci - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-x86_64-linux.tar.gz - buildah manifest add localhost/nix-ci docker-archive:dist/nix-ci-aarch64-linux.tar.gz - buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:''${CI_COMMIT_SHORT_SHA} - # branches - if [ -z "$CI_COMMIT_TAG" ]; then - buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:''${NORMALIZED_BRANCH/main/latest} - fi - # tags - if [ -n "$CI_COMMIT_TAG" ]; then - buildah manifest push --all localhost/nix-ci docker://''${CI_REGISTRY_IMAGE}/nix-ci:''${CI_COMMIT_TAG} - fi - '' - ]; - }; - }; - }; - pipelines."default" = { - stages = ["test" "build" "deploy"]; - jobs = { - "test" = { - stage = "test"; - script = [ - "nix run .#tests -- --junit=junit.xml" - ]; - allow_failure = true; - artifacts = { - when = "always"; - reports.junit = "junit.xml"; - }; - }; - "docs" = { - stage = "build"; - script = [ - # sh - '' - nix build .#docs:default - mkdir -p public - cp -r result/. public/ - '' - ]; - artifacts.paths = ["public"]; - }; - "pages" = { - nix.enable = false; - image = "alpine:latest"; - stage = "deploy"; - script = ["true"]; - artifacts.paths = ["public"]; - rules = [ - { - "if" = "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"; - } - ]; - }; - }; - }; - } diff --git a/nix/repo/devShells.nix b/nix/repo/devShells.nix deleted file mode 100644 index 10889f9..0000000 --- a/nix/repo/devShells.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ - cell, - inputs, - ... -}: let - inherit (inputs) pkgs devshell treefmt soonix; - inherit (cell) ci; -in { - default = devshell.mkShell { - imports = [soonix.devshellModule]; - packages = [ - pkgs.nil - (treefmt.mkWrapper pkgs { - programs = { - alejandra.enable = true; - deadnix.enable = true; - statix.enable = true; - mdformat.enable = true; - yamlfmt.enable = true; - }; - settings.formatter = { - yamlfmt.excludes = ["templates/nix-gitlab-ci.yml" ".gitlab-ci.yml"]; - mdformat.command = let - pkg = pkgs.python3.withPackages (p: [ - p.mdformat - p.mdformat-mkdocs - ]); - in "${pkg}/bin/mdformat"; - }; - }) - ]; - - soonix.hooks."ci" = ci.soonix; - }; -} diff --git a/nix/repo/docs.nix b/nix/repo/docs.nix deleted file mode 100644 index 052b177..0000000 --- a/nix/repo/docs.nix +++ /dev/null @@ -1,80 +0,0 @@ -{inputs, ...}: let - inherit (inputs) pkgs cilib doclib; - - optionsDoc = doclib.mkOptionDocs { - module = cilib.modules.nixCiSubmodule; - roots = [ - { - url = "https://gitlab.com/TECHNOFAB/nix-gitlab-ci/-/blob/main/lib"; - path = "${inputs.self}/lib"; - } - ]; - }; - optionsDocs = pkgs.runCommand "options-docs" {} '' - mkdir -p $out - ln -s ${optionsDoc} $out/options.md - ''; -in - (doclib.mkDocs { - docs."default" = { - base = "${inputs.self}"; - path = "${inputs.self}/docs"; - material = { - enable = true; - colors = { - primary = "deep orange"; - accent = "orange"; - }; - umami = { - enable = true; - src = "https://analytics.tf/umami"; - siteId = "28f7c904-db22-4c2b-9ee4-ed42e14b6db9"; - domains = ["nix-gitlab-ci.projects.tf"]; - }; - }; - macros = { - enable = true; - includeDir = toString optionsDocs; - }; - config = { - site_name = "Nix-GitLab-CI"; - site_url = "https://nix-gitlab-ci.projects.tf"; - repo_name = "TECHNOFAB/nix-gitlab-ci"; - repo_url = "https://gitlab.com/TECHNOFAB/nix-gitlab-ci"; - extra_css = ["style.css"]; - theme = { - logo = "images/logo.svg"; - icon.repo = "simple/gitlab"; - favicon = "images/logo.svg"; - }; - nav = [ - {"Introduction" = "index.md";} - {"Setup" = "setup.md";} - {"Usage" = "usage.md";} - {"CI/CD Component" = "cicd_component.md";} - {"Environment Variables" = "environment_variables.md";} - {"Caching" = "caching.md";} - {"Multiple Pipelines" = "multi_pipeline.md";} - {"Soonix Integration" = "soonix.md";} - {"Utilities" = "utilities.md";} - {"Kubernetes Runner Example" = "kubernetes_runner.md";} - {"Example Configs" = "examples.md";} - {"Options" = "options.md";} - ]; - markdown_extensions = [ - { - "pymdownx.highlight".pygments_lang_class = true; - } - "pymdownx.inlinehilite" - "pymdownx.snippets" - "pymdownx.superfences" - "pymdownx.escapeall" - "fenced_code" - "admonition" - ]; - }; - }; - }).packages - // { - inherit optionsDocs; - } diff --git a/nix/repo/flake.lock b/nix/repo/flake.lock deleted file mode 100644 index e5029e0..0000000 --- a/nix/repo/flake.lock +++ /dev/null @@ -1,99 +0,0 @@ -{ - "nodes": { - "devshell-lib": { - "locked": { - "dir": "lib", - "lastModified": 1758204313, - "narHash": "sha256-ainbY0Oajb1HMdvy+A8QxF/P5qwcbEzJGEY5pzKdDdc=", - "owner": "rensa-nix", - "repo": "devshell", - "rev": "7d0c4bc78d9f017a739b0c7eb2f4e563118353e6", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "rensa-nix", - "repo": "devshell", - "type": "gitlab" - } - }, - "nixmkdocs-lib": { - "locked": { - "dir": "lib", - "lastModified": 1763481845, - "narHash": "sha256-Bp0+9rDmlPWMcnKqGx+BG4+o5KO8FuDAOvXRnXrm3Fo=", - "owner": "TECHNOFAB", - "repo": "nixmkdocs", - "rev": "73d59093df94a894d25bc4bf71880b6f00faa62f", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "TECHNOFAB", - "repo": "nixmkdocs", - "type": "gitlab" - } - }, - "nixtest-lib": { - "locked": { - "dir": "lib", - "lastModified": 1759340550, - "narHash": "sha256-EH9heYb/nHHzCpUGQGqVQnuyVGQ7D6MVMgJmzNvvmJ8=", - "owner": "TECHNOFAB", - "repo": "nixtest", - "rev": "5a7053afcbb211b9cf8fe87f7892bb9f6b76b678", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "TECHNOFAB", - "repo": "nixtest", - "type": "gitlab" - } - }, - "root": { - "inputs": { - "devshell-lib": "devshell-lib", - "nixmkdocs-lib": "nixmkdocs-lib", - "nixtest-lib": "nixtest-lib", - "soonix-lib": "soonix-lib", - "treefmt-nix": "treefmt-nix" - } - }, - "soonix-lib": { - "locked": { - "dir": "lib", - "lastModified": 1763323017, - "narHash": "sha256-MJyg37d+VMfRoFiVUj16FW+zkEwQXbgK9LoFF/SHoxA=", - "owner": "TECHNOFAB", - "repo": "soonix", - "rev": "078034b01e4eaf1f9436d46721f7cbe0d96eb8b4", - "type": "gitlab" - }, - "original": { - "dir": "lib", - "owner": "TECHNOFAB", - "repo": "soonix", - "type": "gitlab" - } - }, - "treefmt-nix": { - "flake": false, - "locked": { - "lastModified": 1762938485, - "narHash": "sha256-AlEObg0syDl+Spi4LsZIBrjw+snSVU4T8MOeuZJUJjM=", - "owner": "numtide", - "repo": "treefmt-nix", - "rev": "5b4ee75aeefd1e2d5a1cc43cf6ba65eba75e83e4", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "treefmt-nix", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/nix/repo/flake.nix b/nix/repo/flake.nix deleted file mode 100644 index 149ae74..0000000 --- a/nix/repo/flake.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ - inputs = { - devshell-lib.url = "gitlab:rensa-nix/devshell?dir=lib"; - nixtest-lib.url = "gitlab:TECHNOFAB/nixtest?dir=lib"; - soonix-lib.url = "gitlab:TECHNOFAB/soonix?dir=lib"; - nixmkdocs-lib.url = "gitlab:TECHNOFAB/nixmkdocs?dir=lib"; - treefmt-nix = { - url = "github:numtide/treefmt-nix"; - flake = false; - }; - }; - outputs = i: - i - // { - devshell = i.devshell-lib.lib {inherit (i.parent) pkgs;}; - soonix = i.soonix-lib.lib {inherit (i.parent) pkgs;}; - ntlib = i.nixtest-lib.lib {inherit (i.parent) pkgs;}; - doclib = i.nixmkdocs-lib.lib {inherit (i.parent) pkgs;}; - cilib = import "${i.parent.self}/lib" {inherit (i.parent) pkgs;}; - treefmt = import i.treefmt-nix; - }; -} diff --git a/nix/repo/tests.nix b/nix/repo/tests.nix deleted file mode 100644 index 62c0833..0000000 --- a/nix/repo/tests.nix +++ /dev/null @@ -1,10 +0,0 @@ -{inputs, ...}: let - inherit (inputs) pkgs ntlib cilib; -in { - tests = ntlib.mkNixtest { - modules = ntlib.autodiscover {dir = "${inputs.self}/tests";}; - args = { - inherit pkgs ntlib cilib; - }; - }; -} diff --git a/templates/nix-gitlab-ci.yml b/templates/nix-gitlab-ci.yml index 57c56d9..425d203 100644 --- a/templates/nix-gitlab-ci.yml +++ b/templates/nix-gitlab-ci.yml @@ -1,114 +1,64 @@ spec: inputs: - cache_strategy: + image_tag: type: string - description: | - auto (default) | none | runner | cachix | attic - Sets the default caching strategy. - - "auto": dynamically selects the best strategy for every job based on env variables - - "none": disables caching - - "runner", "cachix" & "attic": forces every job to use this strategy - - Can be overridden by setting NIX_CI_CACHE_STRATEGY in the pipeline variables. - default: "auto" + description: "latest | latest-cachix | latest-attic etc." + default: latest cache_files: type: array description: | Files to use as the cache key for the generated pipeline yaml. - If you use "ci.nix" to define CI, add that here for example. - Note that max 2 items are allowed in cache:key:files, so use something like - ["flake.*", "ci.nix"] f. ex. to match flake.lock, flake.nix and ci.nix. + If you use "ci.nix" to define CI, add that here for example default: ["flake.nix", "flake.lock"] - version: + disable_cache: type: string description: | - Which version of the Nix CI image to use. Using a tag/version is recommended. - stage_build: - type: string - description: The CI stage for building the dynamic pipeline. - default: build - stage_trigger: - type: string - description: The CI stage for triggering the dynamic pipeline. - default: trigger + Disables any caching provided by this component. Set to any non-empty value to disable caching. + default: "" --- stages: - - $[[ inputs.stage_build ]] - - $[[ inputs.stage_trigger ]] + - build + - trigger variables: - # These can be overriden, see https://docs.gitlab.com/ci/variables/#cicd-variable-precedence - # which image should be used by default. - NIX_CI_IMAGE: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.version ]] - # default cache stategy - NIX_CI_CACHE_STRATEGY: $[[ inputs.cache_strategy ]] + NIX_CI_DISABLE_CACHE: "$[[ inputs.disable_cache ]]" nix-ci:build: - stage: $[[ inputs.stage_build ]] - image: $NIX_CI_IMAGE + stage: build + image: registry.gitlab.com/technofab/nix-gitlab-ci/nix-ci:$[[ inputs.image_tag ]] cache: - key: files: $[[ inputs.cache_files ]] paths: - - .nix-ci-pipelines/ + - generated-gitlab-ci.yml - key: nix paths: - .nix-cache/ before_script: - - | - # if no explicit pipeline is requested - if [[ -z "${NIX_CI_PIPELINE_NAME:-}" ]]; then - # if regex matches, use pipeline "default", otherwise $CI_PIPELINE_SOURCE - [[ "${CI_PIPELINE_SOURCE}" =~ ${NIX_CI_DEFAULT_SOURCES:-.*} ]] \ - && NIX_CI_PIPELINE_NAME="default" \ - || NIX_CI_PIPELINE_NAME="$CI_PIPELINE_SOURCE"; - fi - echo "NIX_CI_GENERATED_PIPELINE_NAME=$NIX_CI_PIPELINE_NAME" >> trigger.env - echo "ORIGINAL_CI_PIPELINE_SOURCE=$CI_PIPELINE_SOURCE" >> trigger.env - # inheritance of pipeline variables is a bit weird, so explicitly override them - # (ctx: setting any of these in the project variables would only apply correctly - # in this pipeline, not the child pipeline, instead weirdly enough the default - # variables above are used). If any other variables are added at the top, add them - # here aswell - echo "NIX_CI_IMAGE=$NIX_CI_IMAGE" >> trigger.env - echo "NIX_CI_CACHE_STRATEGY=$NIX_CI_CACHE_STRATEGY" >> trigger.env - - mkdir -p .nix-ci-pipelines/ - # generated-gitlab-ci.yml exists in the cache - [[ -f ".nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml" ]] && export CACHED=true && echo "A cached pipeline file exists (skip cache with NIX_CI_FORCE_BUILD)" || true - # allow the user to manually skip the cache (when the key files are not correctly configured etc.) - [[ -n "$NIX_CI_FORCE_BUILD" ]] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_FORCE_BUILD)" || true - - # only setup when we need to generate the pipeline yaml - if [[ -z "$CACHED" ]]; then - source setup_nix_ci; - fi + # generated-gitlab-ci.yml exists in the cache + - '[ -f "generated-gitlab-ci.yml" ] && export CACHED=true && echo "Using cached pipeline file (skip cache with NIX_CI_SKIP_CACHE)" || true' + # allow the user to manually skip the cache (when the key files are not correctly configured etc.) + - '[ -n "$NIX_CI_SKIP_CACHE" ] && unset CACHED && echo "Caching skipped for this job (through NIX_CI_SKIP_CACHE)" || true' + # only setup when we need to generate the pipeline yaml + - 'if [ -z "$CACHED" ]; then source setup_nix_ci; fi' script: - # build the pipeline if it does not exist in the cache - - > - if [[ -z "$CACHED" ]]; then - nix build .#gitlab-ci:pipeline:${NIX_CI_PIPELINE_NAME} && install result .nix-ci-pipelines/${NIX_CI_PIPELINE_NAME}.yml; - fi + # build the generated-gitlab-ci.yml if it does not exist in the cache + - 'if [ -z "$CACHED" ]; then nix build .#gitlab-ci-config && install result generated-gitlab-ci.yml; fi' after_script: + # NOTE: environment variables of before_script and script don't exist here anymore + # # save to binary cache or Gitlab CI cache only if we actually built something # check if /tmp/nix-store-before exists as $CACHED never exists here and the file only exists if "setup_nix_ci" is called - - | - if [[ -f "/tmp/nix-store-before" ]]; then - finalize_nix_ci; - fi + - 'if [ -f "/tmp/nix-store-before" ]; then finalize_nix_ci; fi' artifacts: paths: - - .nix-ci-pipelines/ - reports: - dotenv: trigger.env - + - generated-gitlab-ci.yml nix-ci:trigger: - stage: $[[ inputs.stage_trigger ]] + stage: trigger needs: - nix-ci:build trigger: include: - - artifact: .nix-ci-pipelines/${NIX_CI_GENERATED_PIPELINE_NAME}.yml + - artifact: generated-gitlab-ci.yml job: nix-ci:build strategy: depend forward: pipeline_variables: true - diff --git a/tests/cilib_test.nix b/tests/cilib_test.nix deleted file mode 100644 index 3b76fb8..0000000 --- a/tests/cilib_test.nix +++ /dev/null @@ -1,209 +0,0 @@ -{ - pkgs, - cilib, - ntlib, - ... -}: { - suites."CI Lib" = { - pos = __curPos; - tests = let - inherit (cilib) mkJobDeps mkJobRun mkJobPatched mkPipeline; - deps = mkJobDeps { - key = "test"; - job = { - variables.TEST = "${pkgs.curl}"; - }; - nixConfig = { - enable = true; - deps = [pkgs.hello]; - }; - }; - in [ - { - name = "jobDeps"; - type = "script"; - script = - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${deps} "/nix/store" "should contain nix store path" - assert_file_contains ${deps} '-hello-.*/bin:$PATH' "should contain hello" - assert_file_contains ${deps} "export TEST=" "should export TEST" - assert_file_contains ${deps} "curl" "should contain curl" - ''; - } - { - name = "jobRun"; - type = "script"; - script = let - run = mkJobRun { - key = "test"; - job.script = ["hello"]; - jobDeps = deps; - }; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${run}/bin/gitlab-ci-job:test "sandbox-helper" "should contain sandbox-helper" - assert_file_contains ${run}/bin/gitlab-ci-job:test "gitlab-ci-job-test-raw" "should contain job name" - assert_file_contains ${run.passthru.actualJobScript} "gitlab-ci-job-deps-test" "should contain job name" - assert_file_contains ${run.passthru.actualJobScript} "Running script..." "should contain 'Running script...'" - assert_file_contains ${run.passthru.actualJobScript} "hello" "should contain hello" - ''; - } - { - name = "jobPatched nix disabled"; - expected = {}; - actual = mkJobPatched { - key = "test"; - pipelineName = "test"; - job = {}; - nixConfig.enable = false; - }; - } - { - name = "jobPatched nix disabled with variables and cache"; - expected = { - variables."HELLO" = "world"; - cache = [{key = "example";}]; - }; - actual = mkJobPatched { - key = "test"; - pipelineName = "test"; - job = { - variables."HELLO" = "world"; - cache = [{key = "example";}]; - }; - nixConfig.enable = false; - }; - } - { - name = "jobPatched without runner cache"; - expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - }; - actual = mkJobPatched { - key = "test"; - pipelineName = "test"; - job = {}; - nixConfig = { - enable = true; - enableRunnerCache = false; - }; - }; - } - { - name = "jobPatched with runner cache"; - expected = { - after_script = ["finalize_nix_ci"]; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - cache = [ - { - key = "test"; - paths = [".nix-cache/"]; - } - ]; - variables."NIX_CI_CACHE_STRATEGY" = "runner"; - }; - actual = mkJobPatched { - key = "test"; - pipelineName = "test"; - job = {}; - nixConfig = { - enable = true; - enableRunnerCache = true; - runnerCacheKey = "test"; - }; - }; - } - { - name = "mkPipeline empty"; - expected = {}; - actual = - (mkPipeline { - name = "test"; - nixConfig = {}; - pipeline.jobs = {}; - }).finalConfig; - } - { - name = "mkPipeline empty packages"; - type = "script"; - script = let - pipeline = - ntlib.helpers.toJsonFile - (mkPipeline { - name = "test"; - nixConfig = {}; - pipeline.jobs = {}; - }).packages; - in - # sh - '' - set -euo pipefail - ${ntlib.helpers.path (with pkgs; [jq gnugrep coreutils])} - echo "two keys, one json one pretty" - jq 'keys | length == 2' "${pipeline}" | grep -q true - echo "key[0] is exactly 'gitlab-ci:pipeline:test'" - jq -r 'keys[0]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test" - echo "key[1] is exactly 'gitlab-ci:pipeline:test:pretty'" - jq -r 'keys[1]' "${pipeline}" | grep -qx "gitlab-ci:pipeline:test:pretty" - echo "value contains '/nix/store/'" - jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "/nix/store/" - echo "value contains 'gitlab-ci-test.yml'" - jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}" | grep -q "gitlab-ci-test.yml" - echo "file only contains '{}'" - [[ "$(cat $(jq -r '.["gitlab-ci:pipeline:test"]' "${pipeline}"))" == "{}" ]] - ''; - } - { - name = "ignore store paths in variables with nix disabled"; - expected = { - stages = ["test"]; - test = { - stage = "test"; - variables."TEST" = "${pkgs.hello}"; - }; - }; - actual = - (mkPipeline { - name = "test"; - nixConfig.enable = false; - pipeline = { - stages = ["test"]; - jobs.test = { - stage = "test"; - variables."TEST" = "${pkgs.hello}"; - }; - }; - }).finalConfig; - } - { - # it doesn't make much sense to have any nix store path in variables, but we ignore it for global variables - name = "ignore store paths in global variables"; - expected = { - variables = { - HELLO = "world"; - CURL = toString pkgs.curl; - }; - }; - actual = - (mkPipeline { - name = "test"; - nixConfig.enable = true; - pipeline = { - variables = { - HELLO = "world"; - CURL = toString pkgs.curl; - }; - jobs = {}; - }; - }).finalConfig; - } - ]; - }; -} diff --git a/tests/fixtures/flake_parts/flake.lock b/tests/fixtures/flake_parts/flake.lock deleted file mode 100644 index 15b9835..0000000 --- a/tests/fixtures/flake_parts/flake.lock +++ /dev/null @@ -1,77 +0,0 @@ -{ - "nodes": { - "flake-parts": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "lastModified": 1754487366, - "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1756636162, - "narHash": "sha256-mBecwgUTWRgClJYqcF+y4O1bY8PQHqeDpB+zsAn+/zA=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "37ff64b7108517f8b6ba5705ee5085eac636a249", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib": { - "locked": { - "lastModified": 1753579242, - "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs", - "systems": "systems" - } - }, - "systems": { - "locked": { - "lastModified": 1689347949, - "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", - "owner": "nix-systems", - "repo": "default-linux", - "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default-linux", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/tests/fixtures/flake_parts/flake.nix b/tests/fixtures/flake_parts/flake.nix deleted file mode 100644 index cca696f..0000000 --- a/tests/fixtures/flake_parts/flake.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ - outputs = { - flake-parts, - systems, - ... - } @ inputs: - flake-parts.lib.mkFlake {inherit inputs;} { - imports = [ - "@repo_path@/lib/flakeModule.nix" - ]; - systems = import systems; - flake = {}; - perSystem = _: { - ci = { - config = { - # true is already default, just for testing - nixJobsByDefault = true; - }; - pipelines = { - "default" = { - stages = ["example"]; - jobs."example" = { - stage = "example"; - script = ["echo hello world"]; - }; - }; - "test".jobs."example" = { - stage = ".pre"; - script = ["echo hello world"]; - }; - }; - }; - }; - }; - - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - flake-parts.url = "github:hercules-ci/flake-parts"; - systems.url = "github:nix-systems/default-linux"; - }; -} diff --git a/tests/flake_parts_test.nix b/tests/flake_parts_test.nix deleted file mode 100644 index e998f45..0000000 --- a/tests/flake_parts_test.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ - pkgs, - ntlib, - ... -}: { - suites."flake-parts" = { - pos = __curPos; - tests = [ - { - name = "flakeModule"; - type = "script"; - script = - # sh - '' - ${ntlib.helpers.scriptHelpers} - ${ntlib.helpers.path (with pkgs; [coreutils nix gnused gnugrep jq])} - export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt - export NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt - repo_path=${../.} - - cp ${./fixtures/flake_parts}/* . - # import from the absolute path above, is easier than trying to figure out the repo path etc. - sed -i -e "s|@repo_path@|$repo_path|" flake.nix - - # NOTE: --impure is required since importing modules from absolute paths is not allowed in pure mode - nix build --impure .#gitlab-ci:pipeline:default - assert "-f result" "should exist" - assert_file_contains "result" "finalize_nix_ci" - jq '.' result # check if valid json just to be sure - - nix build --impure .#gitlab-ci:pipeline:default:pretty - assert "-f result" "should exist" - assert_file_contains "result" " - finalize_nix_ci" - ''; - } - ]; - }; -} diff --git a/tests/helpers_test.nix b/tests/helpers_test.nix deleted file mode 100644 index 4d7e7d4..0000000 --- a/tests/helpers_test.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ - pkgs, - cilib, - ... -}: { - suites."Helpers" = { - pos = __curPos; - tests = let - inherit (cilib) helpers; - in [ - { - name = "appendToAfterScript"; - expected = { - after_script = ["echo after_script" "finalize_nix_ci"]; - }; - actual = helpers.appendToAfterScript ["finalize_nix_ci"] { - after_script = ["echo after_script"]; - }; - } - { - name = "prependToBeforeScript"; - expected = { - before_script = ["setup_nix_ci" "echo before_script"]; - }; - actual = helpers.prependToBeforeScript ["setup_nix_ci"] { - before_script = ["echo before_script"]; - }; - } - { - name = "toYaml"; - expected = ''{"hello":"world"}''; - actual = builtins.readFile (helpers.toYaml "test" {hello = "world";}); - } - { - name = "filterAttrsRec"; - expected = {world = "world";}; - actual = helpers.filterAttrsRec (_n: v: v != null) { - hello = null; - world = "world"; - }; - } - { - name = "filterJobVariables with store paths"; - expected = { - HELLO = "${pkgs.hello}"; - MULTIPLE = "${pkgs.hello}:${pkgs.hello}"; - }; - actual = helpers.filterJobVariables true { - variables = { - HELLO = "${pkgs.hello}"; - WORLD = "world"; - MULTIPLE = "${pkgs.hello}:${pkgs.hello}"; - }; - }; - } - { - name = "filterJobVariables without store paths"; - expected = {WORLD = "world";}; - actual = helpers.filterJobVariables false { - variables = { - HELLO = "${pkgs.hello}"; - WORLD = "world"; - }; - }; - } - ]; - }; -} diff --git a/tests/modules_test.nix b/tests/modules_test.nix deleted file mode 100644 index 8ccbd83..0000000 --- a/tests/modules_test.nix +++ /dev/null @@ -1,103 +0,0 @@ -{ - pkgs, - cilib, - ntlib, - ... -}: { - suites."Modules" = { - pos = __curPos; - tests = let - simplePipeline = cilib.mkCI { - pipelines."test" = { - stages = ["test"]; - jobs."test" = { - stage = "test"; - script = ["echo hello world"]; - }; - }; - }; - in [ - { - name = "empty pipelines"; - expected = {}; - actual = - (cilib.mkCI {}).pipelines; - } - { - name = "empty packages"; - expected = {}; - actual = - (cilib.mkCI {}).packages; - } - { - name = "simple pipeline"; - expected = { - stages = [".pre" "test" ".post"]; - "test" = { - image = "$NIX_CI_IMAGE"; - stage = "test"; - before_script = ["source setup_nix_ci \"gitlab-ci:pipeline:test:job-deps:test\""]; - script = ["echo hello world"]; - after_script = ["finalize_nix_ci"]; - }; - }; - actual = simplePipeline.pipelines."test".finalConfig; - } - { - name = "simple pipeline yaml"; - type = "script"; - script = let - package = simplePipeline.packages."gitlab-ci:pipeline:test"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${package} 'gitlab-ci:pipeline:test:job-deps:test' - assert_file_contains ${package} 'finalize_nix_ci' - assert_file_contains ${package} 'echo hello world' - ''; - } - { - name = "dont fail on store paths"; - type = "script"; - script = let - package = - (cilib.mkCI { - pipelines."test" = { - variables.EXAMPLE = "${pkgs.hello}"; - }; - }).packages."gitlab-ci:pipeline:test"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${package} '[".pre",".post"]' - assert_file_contains ${package} '"EXAMPLE":"/nix/store/.*-hello-.*"' - ''; - } - { - name = "correctly inject variables containing nix store paths at runtime"; - type = "script"; - script = let - package = - (cilib.mkCI { - pipelines."test".jobs."test" = { - stage = ".pre"; - variables.EXAMPLE = "${pkgs.hello}"; - script = []; - }; - }).packages."gitlab-ci:pipeline:test:job-deps:test"; - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${package} 'export PATH=":$PATH";' - assert_file_contains ${package} 'export EXAMPLE="/nix/store/.*-hello-.*"' - ''; - } - ]; - }; -} diff --git a/tests/soonix_test.nix b/tests/soonix_test.nix deleted file mode 100644 index c76edb4..0000000 --- a/tests/soonix_test.nix +++ /dev/null @@ -1,77 +0,0 @@ -{ - lib, - cilib, - ... -}: let - inherit (lib) trimWith; -in { - suites."Soonix" = { - pos = __curPos; - tests = let - version = trimWith { - start = true; - end = true; - } (builtins.readFile ../lib/VERSION); - in [ - { - name = "default soonix config"; - expected = { - data.include = [ - { - component = "gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-gitlab-ci@${version}"; - inputs.version = version; - } - ]; - generator = "nix"; - opts.format = "yaml"; - hook = { - mode = "copy"; - gitignore = false; - }; - output = ".gitlab-ci.yml"; - }; - actual = - (cilib.mkCI { - config.soonix = {}; - }).soonix; - } - { - name = "custom soonix config"; - expected = { - data = { - include = [ - { - component = "gitlab.com/example/nix-gitlab-ci/nix-gitlab-ci@abc"; - inputs = { - version = "abc"; - hello = "world"; - }; - } - ]; - "example".script = ["hello"]; - }; - generator = "nix"; - opts.format = "yaml"; - hook = { - mode = "copy"; - gitignore = false; - }; - output = ".gitlab-ci.yml"; - }; - actual = - (cilib.mkCI { - config.soonix = { - componentVersion = "abc"; - componentUrl = "gitlab.com/example/nix-gitlab-ci/nix-gitlab-ci"; - componentInputs = { - hello = "world"; - }; - extraData = { - "example".script = ["hello"]; - }; - }; - }).soonix; - } - ]; - }; -} diff --git a/tests/utils_test.nix b/tests/utils_test.nix deleted file mode 100644 index 2e319b1..0000000 --- a/tests/utils_test.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ - pkgs, - ntlib, - cilib, - ... -}: { - suites."Utils" = { - pos = __curPos; - tests = [ - { - name = "commitAndPushFiles"; - type = "script"; - script = let - inherit (cilib) utils; - job = ntlib.helpers.toJsonFile ( - utils.commitAndPushFiles { - message = "hello world"; - files = ["a.md" "b.txt"]; - } {} - ); - in - # sh - '' - ${ntlib.helpers.path [pkgs.gnugrep]} - ${ntlib.helpers.scriptHelpers} - assert_file_contains ${job} 'git commit -m \\"hello world\\"' - assert_file_contains ${job} 'git add a.md b.txt' - ''; - } - ]; - }; -}