commit 0df36ad437cb4e6f3ca3577987add2b9616bd8f1 Author: technofab Date: Sun Jan 14 16:45:07 2024 +0100 chore: initial commit diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..a2d9328 --- /dev/null +++ b/.envrc @@ -0,0 +1,4 @@ +if ! use flake . --impure +then + echo "devenv could not be build. The devenv environment was not loaded. Make the necessary changes to flake.nix and hit enter to try again." >&2 +fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a8dde4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea +.devenv +.direnv +.pre-commit-config.yaml +result diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..2c641d7 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,22 @@ +include: + - gitlab-ci.yml + +stages: + - build + - trigger + +build:image: + stage: build + image: nixpkgs/nix-flakes:latest + before_script: + - nix profile install nixpkgs#skopeo + - export PATH="$PATH:$HOME/.nix-profile/bin" + - mkdir -p /var/tmp + script: + - nix build .#image + - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" + "docker-archive:result" + "docker://$CI_REGISTRY_IMAGE/nix-ci:latest" + - skopeo --insecure-policy copy --dest-creds "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" + "docker-archive:result" + "docker://$CI_REGISTRY_IMAGE/nix-ci:$(date +"%m-%d-%y")" diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..1db7b50 --- /dev/null +++ b/flake.lock @@ -0,0 +1,424 @@ +{ + "nodes": { + "attic": { + "inputs": { + "crane": "crane", + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1704159259, + "narHash": "sha256-OOEFibN9JQBepVeqxSNSBr6JUmdoZiH263ogms2jk0k=", + "owner": "zhaofengli", + "repo": "attic", + "rev": "e6bedf1869f382cfc51b69848d6e09d51585ead6", + "type": "github" + }, + "original": { + "owner": "zhaofengli", + "repo": "attic", + "type": "github" + } + }, + "crane": { + "inputs": { + "nixpkgs": [ + "attic", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1702918879, + "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=", + "owner": "ipetkov", + "repo": "crane", + "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "devenv": { + "inputs": { + "flake-compat": "flake-compat_2", + "nix": "nix", + "nixpkgs": "nixpkgs_2", + "pre-commit-hooks": [ + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1705094340, + "narHash": "sha256-T7d1d5PQXqDbZUnKjRTBf26yTCMSttDQULM8jU4jiro=", + "owner": "cachix", + "repo": "devenv", + "rev": "96b49eb381779bdd4f41ca176a762867c31db5da", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1704982712, + "narHash": "sha256-2Ptt+9h8dczgle2Oo6z5ni5rt/uLMG47UFTR1ry/wgg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "07f6395285469419cf9d078f59b5b49993198c00", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1703887061, + "narHash": "sha256-gGPa9qWNc6eCXT/+Z5/zMkyYOuRZqeFZBDbopNZQkuY=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "43e1aa1308018f37118e34d3a9cb4f5e75dc11d5", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "lowdown-src": { + "flake": false, + "locked": { + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, + "nix": { + "inputs": { + "lowdown-src": "lowdown-src", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1676545802, + "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "owner": "domenkozar", + "repo": "nix", + "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "relaxed-flakes", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1702539185, + "narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1703961334, + "narHash": "sha256-M1mV/Cq+pgjk0rt6VxoyyD+O8cOUiai8t9Q6Yyq4noY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b0d36bd0a420ecee3bc916c91886caca87c894e9", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1702780907, + "narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-stable_2": { + "locked": { + "lastModified": 1704874635, + "narHash": "sha256-YWuCrtsty5vVZvu+7BchAxmcYzTMfolSPP5io8+WYCg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3dc440faeee9e889fe2d1b4d25ad0f430d449356", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1678875422, + "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1705204527, + "narHash": "sha256-WVz9WdaFBhAwO/7A+HlW8HPJ4VQ8QnpCD1WZAcAPneo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "dd5621df6dcb90122b50da5ec31c411a0de3e538", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1704842529, + "narHash": "sha256-OTeQA+F8d/Evad33JMfuXC89VMetQbsU4qcaePchGr4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "eabe8d3eface69f5bb16c18f8662a702f50c20d5", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": "flake-compat_3", + "flake-utils": "flake-utils_2", + "gitignore": "gitignore", + "nixpkgs": "nixpkgs_4", + "nixpkgs-stable": "nixpkgs-stable_2" + }, + "locked": { + "lastModified": 1705229514, + "narHash": "sha256-itILy0zimR/iyUGq5Dgg0fiW8plRDyxF153LWGsg3Cw=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "ffa9a5b90b0acfaa03b1533b83eaf5dead819a05", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "attic": "attic", + "devenv": "devenv", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs_3", + "pre-commit-hooks": "pre-commit-hooks", + "systems": "systems_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1689347949, + "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", + "owner": "nix-systems", + "repo": "default-linux", + "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default-linux", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..429983b --- /dev/null +++ b/flake.nix @@ -0,0 +1,112 @@ +{ + outputs = { + self, + flake-parts, + systems, + nixpkgs, + devenv, + ... + } @ inputs: + flake-parts.lib.mkFlake {inherit inputs;} { + imports = [ + inputs.devenv.flakeModule + ./flakeModule.nix + ]; + systems = import systems; + flake = { + flakeModule = ./flakeModule.nix; + }; + perSystem = { + pkgs, + system, + inputs', + ... + }: { + formatter = pkgs.alejandra; + devenv.shells = { + default = { + packages = [pkgs.dive pkgs.skopeo]; + + pre-commit = { + hooks = { + alejandra.enable = true; + }; + }; + }; + }; + ci = { + stages = ["test"]; + default = { + retry = { + max = 2; + when = "runner_system_failure"; + }; + }; + jobs = { + "test" = { + stage = "test"; + # wait an hour so the image builds + start_in = "1 hour"; + deps = [pkgs.hello pkgs.curl]; + script = [ + "hello" + "curl google.de" + ]; + }; + "test-non-nix" = { + nix = false; + stage = "test"; + image = "alpine:latest"; + script = [ + "echo \"This job will not be modified to use nix\"" + ]; + }; + }; + }; + + packages = { + image = pkgs.dockerTools.buildImage { + name = "nix-gitlab-ci"; + fromImage = pkgs.dockerTools.pullImage { + imageName = "nixpkgs/nix-unstable-static"; + imageDigest = "sha256:8a9138d663d5a0d4c3ab01ee328e7e30dd236119d542df0a0355d66231049228"; + sha256 = "1ynn0xklh3bjqqrm1438fw029pwzsrxr0645nbyb6bl0x4102rjs"; + finalImageName = "nixpkgs/nix-unstable-static"; + finalImageTag = "latest"; + }; + copyToRoot = pkgs.buildEnv { + name = "image-root"; + paths = [pkgs.gitMinimal pkgs.cachix inputs'.attic.packages.attic-client]; + pathsToLink = ["/bin"]; + }; + }; + }; + }; + }; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + # flake & devenv related + flake-parts.url = "github:hercules-ci/flake-parts"; + systems.url = "github:nix-systems/default-linux"; + devenv = { + url = "github:cachix/devenv"; + inputs.pre-commit-hooks.follows = "pre-commit-hooks"; + }; + pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + attic.url = "github:zhaofengli/attic"; + }; + + nixConfig = { + extra-substituters = [ + "https://cache.nixos.org/" + "https://nix-community.cachix.org" + ]; + + extra-trusted-public-keys = [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + ]; + }; +} diff --git a/flakeModule.nix b/flakeModule.nix new file mode 100644 index 0000000..04a5df9 --- /dev/null +++ b/flakeModule.nix @@ -0,0 +1,165 @@ +{ + flake-parts-lib, + lib, + inputs, + ... +}: { + options.perSystem = flake-parts-lib.mkPerSystemOption ( + { + config, + pkgs, + system, + ... + }: let + cfg = config.ci.config; + + subType = options: lib.types.submodule {inherit options;}; + + configType = with lib; + subType { + default-nix-image = mkOption { + type = types.str; + default = "registry.gitlab.com/TECHNOFAB/nix-gitlab-ci/nix-ci:latest"; + description = "The image to use on nix jobs"; + }; + nix-jobs-per-default = mkOption { + type = types.bool; + default = true; + description = "Handle jobs nix-based by default or via opt-in (in job set nix = true) if false"; + }; + }; + jobType = with lib; + subType { + # nix ci opts + nix = mkOption { + type = types.bool; + default = cfg.nix-jobs-per-default; + }; + deps = mkOption { + type = types.listOf types.package; + default = []; + }; + # gitlab opts + variables = mkOption { + type = types.attrs; + default = {}; + }; + before_script = mkOption { + type = types.listOf types.str; + default = []; + }; + script = mkOption { + type = types.listOf types.str; + default = []; + }; + after_script = mkOption { + type = types.listOf types.str; + default = []; + }; + stage = mkOption { + type = types.str; + }; + image = mkOption { + type = types.str; + default = cfg.default-nix-image; + }; + }; + in { + options = with lib; { + ci = mkOption { + type = subType { + config = mkOption { + type = configType; + description = '' + Configuration options for the nix part itself + ''; + default = {}; + }; + image = mkOption { + type = types.nullOr types.str; + default = null; + }; + variables = mkOption { + type = types.attrs; + default = {}; + }; + default = mkOption { + type = types.attrs; + default = {}; + }; + stages = mkOption { + type = types.listOf types.str; + default = []; + }; + jobs = mkOption { + type = types.lazyAttrsOf jobType; + default = {}; + }; + }; + description = '' + Generate a Gitlab CI configuration which can be used to trigger a child pipeline. + This will inject code which pre-downloads the nix deps before each job and adds them to PATH. + ''; + default = {}; + }; + }; + + config.packages = let + toYaml = (pkgs.formats.yaml {}).generate; + mapAttrs = cb: set: builtins.listToAttrs (builtins.map (key: cb key (builtins.getAttr key set)) (builtins.attrNames set)); + prependToBeforeScript = arr: job: + job + // lib.optionalAttrs job.nix { + before_script = + arr + ++ job.before_script; + }; + + jobs = config.ci.jobs; + rest = builtins.removeAttrs config.ci ["jobs" "config"]; + # this allows us to nix build this to get all the mentioned dependencies from the binary cache + # pro: we don't have to download everything, just the deps for the current job + # before, we just allowed pkgs inside the script string directly, but now with the ability to source this file + # we can support different architectures between runners (eg. the arch of the initial runner does not matter) + jobsMappedForDeps = + mapAttrs (key: job: { + name = "gitlab-ci-job-deps:${key}"; + value = pkgs.writeShellScript "gitlab-ci-jobs:${key}" '' + export PATH="${lib.makeBinPath job.deps}:$PATH"; + ''; + }) + jobs; + # allows the user to directly run the script + jobsMappedForScript = + mapAttrs (key: job: { + name = "gitlab-ci-job:${key}"; + value = pkgs.writeShellScriptBin "gitlab-ci-jobs:${key}" (lib.strings.concatLines (job.before_script ++ job.script ++ job.after_script)); + }) + jobs; + # build the deps specific for this job before anything, this way the deps should be fetched from the cache + jobsPatched = + mapAttrs (key: job: { + name = key; + value = builtins.removeAttrs ( + (prependToBeforeScript [ + "echo -e \"\\e[0Ksection_start:`date +%s`:nix_deps[collapsed=true]\\r\\e[0KDownload Nix deps for job\"" + "nix build .#gitlab-ci-job-deps:${key}" + "source $(readlink -f result)" + "echo -e \"\\e[0Ksection_end:`date +%s`:nix_deps\\r\\e[0K\"" + ] + job) + // lib.optionalAttrs job.nix { + image = job.image; + } + ) ["nix" "deps"]; + }) + jobs; + in + { + gitlab-ci-config = toYaml "generated-gitlab-ci.yml" (rest // jobsPatched); + } + // jobsMappedForDeps + // jobsMappedForScript; + } + ); +} diff --git a/gitlab-ci.yml b/gitlab-ci.yml new file mode 100644 index 0000000..75913ec --- /dev/null +++ b/gitlab-ci.yml @@ -0,0 +1,36 @@ + +variables: + # cachix | attic + NIX_CI_CACHE_TYPE: cachix + +stages: + - build + - trigger + +nix-ci:build: + stage: build + image: nixos/nix:latest + before_script: + - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf + - echo "accept-flake-config = true" >> /etc/nix/nix.conf + script: + # build the generated-gitlab-ci.yml + - nix build .#gitlab-ci-config + - install result generated-gitlab-ci.yml + after_script: + # upload to binary cache + - echo $(nix-store --query --references result) + # TODO: upload deps using cachix or attic + artifacts: + paths: + - generated-gitlab-ci.yml + +nix-ci:trigger: + stage: trigger + needs: + - nix-ci:build + trigger: + include: + - artifact: generated-gitlab-ci.yml + job: nix-ci:build + strategy: depend