feat: rewrite how nixlets work

This commit is contained in:
TECHNOFAB 2024-05-08 17:09:17 +00:00
parent ba81fba6b2
commit 0cde19e51f
36 changed files with 494 additions and 125 deletions

18
ci.nix
View file

@ -4,7 +4,7 @@
...
}: {
ci = {
stages = ["check"];
stages = ["check" "upload"];
default = {
retry = {
max = 2;
@ -14,10 +14,26 @@
jobs = {
"check" = {
stage = "check";
before_script = [
"nix flake prefetch path:lib"
];
script = [
"nix flake check --impure"
];
};
"upload" = {
stage = "upload";
rules = [
{"if" = ''$CI_COMMIT_REF_NAME == "main"'';}
];
variables.AUTH_HEADER = "JOB-TOKEN: \${CI_JOB_TOKEN}";
before_script = [
"nix flake prefetch path:lib"
];
script = [
"nix run .#upload --impure"
];
};
};
};
}

123
flake.lock generated
View file

@ -172,6 +172,22 @@
}
},
"flake-compat_6": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_7": {
"flake": false,
"locked": {
"lastModified": 1696426674,
@ -258,7 +274,7 @@
},
"flake-utils_3": {
"inputs": {
"systems": "systems_4"
"systems": "systems_5"
},
"locked": {
"lastModified": 1701680307,
@ -340,6 +356,30 @@
"type": "github"
}
},
"kubenix_2": {
"inputs": {
"flake-compat": "flake-compat_6",
"nixpkgs": [
"nixlet-lib",
"nixpkgs"
],
"systems": "systems_4",
"treefmt": "treefmt_2"
},
"locked": {
"lastModified": 1709908607,
"narHash": "sha256-cG5PftryvQT5vTtRJGGsnfFgdobaUmD2klVVhzddFiU=",
"owner": "TECHNOFAB11",
"repo": "kubenix",
"rev": "ddfdd8d0903acff4a0a52b84e4395da1778dcc4c",
"type": "github"
},
"original": {
"owner": "TECHNOFAB11",
"repo": "kubenix",
"type": "github"
}
},
"lowdown-src": {
"flake": false,
"locked": {
@ -444,6 +484,22 @@
"type": "github"
}
},
"nixlet-lib": {
"inputs": {
"kubenix": "kubenix_2",
"nixpkgs": "nixpkgs_6"
},
"locked": {
"lastModified": 1,
"narHash": "sha256-DPYdVXcXcHnC9Js4ff28Pbux/uPE8eBqHGrmBaYMhmg=",
"path": "lib",
"type": "path"
},
"original": {
"path": "lib",
"type": "path"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1678875422,
@ -641,6 +697,22 @@
}
},
"nixpkgs_6": {
"locked": {
"lastModified": 1715037484,
"narHash": "sha256-OUt8xQFmBU96Hmm4T9tOWTu4oCswCzoVl+pxSq/kiFc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ad7efee13e0d216bf29992311536fce1d3eefbef",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_7": {
"locked": {
"lastModified": 1709386671,
"narHash": "sha256-VPqfBnIJ+cfa78pd4Y5Cr6sOWVW8GYHRVucxJGmRf8Q=",
@ -656,7 +728,7 @@
"type": "github"
}
},
"nixpkgs_7": {
"nixpkgs_8": {
"locked": {
"lastModified": 1704842529,
"narHash": "sha256-OTeQA+F8d/Evad33JMfuXC89VMetQbsU4qcaePchGr4=",
@ -696,10 +768,10 @@
},
"pre-commit-hooks_2": {
"inputs": {
"flake-compat": "flake-compat_6",
"flake-compat": "flake-compat_7",
"flake-utils": "flake-utils_3",
"gitignore": "gitignore_2",
"nixpkgs": "nixpkgs_7",
"nixpkgs": "nixpkgs_8",
"nixpkgs-stable": "nixpkgs-stable_3"
},
"locked": {
@ -722,9 +794,10 @@
"flake-parts": "flake-parts",
"kubenix": "kubenix",
"nix-gitlab-ci": "nix-gitlab-ci",
"nixpkgs": "nixpkgs_6",
"nixlet-lib": "nixlet-lib",
"nixpkgs": "nixpkgs_7",
"pre-commit-hooks": "pre-commit-hooks_2",
"systems": "systems_5"
"systems": "systems_6"
}
},
"systems": {
@ -772,6 +845,20 @@
}
},
"systems_4": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"id": "systems",
"type": "indirect"
}
},
"systems_5": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
@ -786,7 +873,7 @@
"type": "github"
}
},
"systems_5": {
"systems_6": {
"locked": {
"lastModified": 1689347949,
"narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=",
@ -821,6 +908,28 @@
"repo": "treefmt-nix",
"type": "github"
}
},
"treefmt_2": {
"inputs": {
"nixpkgs": [
"nixlet-lib",
"kubenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1688026376,
"narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
}
},
"root": "root",

View file

@ -2,6 +2,7 @@
outputs = {
self,
flake-parts,
nixlet-lib,
systems,
...
} @ inputs:
@ -11,24 +12,19 @@
inputs.nix-gitlab-ci.flakeModule
];
systems = import systems;
flake = rec {
utils = import ./lib {
inherit (inputs.nixpkgs) lib;
inherit inputs;
};
# █▄░█ █ ▀▄▀ █░░ █▀▀ ▀█▀ █▀
# █░▀█ █ █░█ █▄▄ ██▄ ░█░ ▄█
nixlets = {
# <name> = utils.mkNixlet ./nixlets/<name>;
mosquitto = utils.mkNixlet ./nixlets/mosquitto;
attic = utils.mkNixlet ./nixlets/attic;
postgres = utils.mkNixlet ./nixlets/postgres;
tikv = utils.mkNixlet ./nixlets/tikv;
surrealdb = utils.mkNixlet ./nixlets/surrealdb;
flake = {
# █▄ █ █ ▀▄▀ █ █▀▀ ▀█▀ █▀
# █ ▀█ █ █ █ █▄▄ ██▄ █ ▄█
nixlets = with nixlet-lib; {
mosquitto = mkNixlet ./nixlets/mosquitto;
attic = mkNixlet ./nixlets/attic;
postgres = mkNixlet ./nixlets/postgres;
tikv = mkNixlet ./nixlets/tikv;
surrealdb = mkNixlet ./nixlets/surrealdb;
};
};
perSystem = {
lib,
pkgs,
system,
inputs',
@ -54,11 +50,9 @@
# check if every nixlet successfully renders with default values
checks =
builtins.mapAttrs (
name: nixlet:
self.utils.renderNixlet {
inherit system nixlet;
project = name;
values = {};
_: nixlet:
nixlet.render {
inherit system;
}
)
self.nixlets;
@ -66,14 +60,20 @@
# allow directly building every nixlet with default values
packages =
builtins.mapAttrs (
name: nixlet:
self.utils.renderNixlet {
inherit system nixlet;
project = name;
values = {};
_: nixlet:
nixlet.render {
inherit system;
}
)
self.nixlets;
apps.upload = {
type = "app";
program = pkgs.callPackage nixlet-lib.uploadNixletsToGitlab {
projectId = "55602785";
nixlets = lib.attrValues self.nixlets;
};
};
};
};
@ -94,5 +94,6 @@
url = "github:TECHNOFAB11/kubenix";
inputs.nixpkgs.follows = "nixpkgs";
};
nixlet-lib.url = "path:lib";
};
}

View file

@ -3,11 +3,11 @@
lib,
...
} @ attrs:
with lib; {
with lib; rec {
mkValues = file: {rawValues, ...} @ args:
(lib.evalModules {
specialArgs = {
utils = import ./. attrs;
utils = import ./utils.nix attrs;
};
modules = [
file
@ -23,58 +23,103 @@ with lib; {
})
.config;
mkNestedOption = options:
mkOption {
type = types.submodule {
inherit options;
};
default = {};
};
# wraps mkNixletInner to allow passing either a path or an attrset
mkNixlet = arg:
mkNixletInner (
if (builtins.typeOf arg) == "set"
then arg
else
{path = arg;}
// (
if builtins.pathExists "${arg}/nixlet.nix"
then (import "${arg}/nixlet.nix")
else throw "Nixlet at '${arg}' does not contain nixlet.nix and mkNixlet was called with just a path"
)
);
mkNixlet = path: let
utils = import ./. attrs;
in
{
rawValues,
project,
...
} @ args: {
kubenix,
lib,
...
} @ attrs: let
values = utils.mkValues "${path}/values.nix" args;
in {
imports = [path];
# make values accessible from every imported file
_module.args = {inherit values;};
};
mkNixletInner = {
path,
name,
version ? null,
description ? "",
defaultProject ? null,
...
}: let
# TODO: just like with the values check the args here with the options system?
in {
inherit name version description path;
render = {
system,
project ? defaultProject,
overrides ? ({...}: {}),
values ? {},
}:
assert lib.assertMsg (project != null) "No default project set, please pass a project to the render method"; let
# every nixlet gets "nixlet" as arg with some useful data about itself
nixletArg = {
inherit name project version description;
};
in
(inputs.kubenix.evalModules.${system} {
module = {kubenix, ...}: {
imports = with kubenix.modules; [
k8s
helm
docker
files
({...}: let
finalValues = mkValues "${path}/values.nix" {
rawValues = values;
nixlet = nixletArg;
};
in {
imports = [path];
_module.args.nixlet =
{
values = finalValues;
}
// nixletArg;
})
overrides
];
kubenix.project = project;
};
})
.config
.kubernetes
.resultYAML;
};
renderNixlet = {
system,
project,
nixlet,
values ? {},
overrides ? {...}: {},
fetchNixlet = url: sha256: mkNixlet (builtins.fetchTarball {inherit url sha256;});
uploadNixletsToGitlab = {
pkgs,
projectId,
nixlets,
...
}:
(inputs.kubenix.evalModules.${system} {
module = {kubenix, ...}: {
imports = with kubenix.modules; [
k8s
helm
docker
files
(nixlet {
# all these args are available in values.nix
inherit project;
rawValues = values;
})
overrides
];
kubenix.project = project;
};
})
.config
.kubernetes
.resultYAML;
pkgs.writeShellScriptBin "nixlets-upload" (
''
if [[ -z "$AUTH_HEADER" ]]; then
echo "Must provide AUTH_HEADER environment variable!" 1>&2
exit 1
fi
''
+ lib.concatStringsSep "\n" (
builtins.map (nixlet:
with nixlet; ''
URL="https://gitlab.com/api/v4/projects/${projectId}/packages/generic/${name}/${version}/${name}.tar.gz"
if curl --output /dev/null --silent --head --fail --header "$AUTH_HEADER" $URL; then
echo "> Skipped ${name}@${version} because it already exists in the Package Registry"
else
echo "> Uploading new version ${name}@${version}"
${pkgs.gnutar}/bin/tar -czf /tmp/${name}.tar.gz --mode='u+rwX' -C ${path} .
${pkgs.curl}/bin/curl --header "$AUTH_HEADER" --upload-file "/tmp/${name}.tar.gz" "$URL"; echo;
${pkgs.coreutils}/bin/rm -f /tmp/${nixlet.name}.tar.gz
echo "> Finished ${name}@${version}, see above"
fi
'')
nixlets
)
);
}

102
lib/flake.lock generated Normal file
View file

@ -0,0 +1,102 @@
{
"nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"kubenix": {
"inputs": {
"flake-compat": "flake-compat",
"nixpkgs": [
"nixpkgs"
],
"systems": "systems",
"treefmt": "treefmt"
},
"locked": {
"lastModified": 1709908607,
"narHash": "sha256-cG5PftryvQT5vTtRJGGsnfFgdobaUmD2klVVhzddFiU=",
"owner": "TECHNOFAB11",
"repo": "kubenix",
"rev": "ddfdd8d0903acff4a0a52b84e4395da1778dcc4c",
"type": "github"
},
"original": {
"owner": "TECHNOFAB11",
"repo": "kubenix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1715037484,
"narHash": "sha256-OUt8xQFmBU96Hmm4T9tOWTu4oCswCzoVl+pxSq/kiFc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ad7efee13e0d216bf29992311536fce1d3eefbef",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"kubenix": "kubenix",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"id": "systems",
"type": "indirect"
}
},
"treefmt": {
"inputs": {
"nixpkgs": [
"kubenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1688026376,
"narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

22
lib/flake.nix Normal file
View file

@ -0,0 +1,22 @@
{
description = "Nixlets lib";
outputs = {
self,
nixpkgs,
kubenix,
...
} @ inputs:
import ./. {
inherit (nixpkgs) lib;
inherit inputs;
};
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
kubenix = {
url = "github:TECHNOFAB11/kubenix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
}

10
lib/utils.nix Normal file
View file

@ -0,0 +1,10 @@
{lib, ...}:
with lib; {
mkNestedOption = options:
mkOption {
type = types.submodule {
inherit options;
};
default = {};
};
}

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
configMaps."${values.uniqueName}-config" = {
data = {

View file

@ -1,8 +1,9 @@
{
values,
nixlet,
lib,
...
}: {
}:
with nixlet; {
kubernetes.resources = let
# switch to a StatefulSet if we don't use an external storage configuration
type =

6
nixlets/attic/nixlet.nix Normal file
View file

@ -0,0 +1,6 @@
{
name = "attic";
version = "0.0.1";
description = "Multi-tenant Nix Binary Cache";
defaultProject = "attic";
}

View file

@ -1,8 +1,9 @@
{
values,
nixlet,
lib,
...
}: {
}:
with nixlet; {
kubernetes.resources = {
persistentVolumeClaims."${values.uniqueName}-data".spec = lib.mkIf (!values.externalStorage) {
accessModes = ["ReadWriteOnce"];

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
services."${values.uniqueName}" = {
spec = {

View file

@ -1,10 +1,11 @@
{
lib,
utils,
project,
nixlet,
...
}:
with lib; {
with lib;
with nixlet; {
# for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml
options = {
replicaCount = mkOption {

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
configMaps."${values.uniqueName}-config" = {
data = {

View file

@ -0,0 +1,6 @@
{
name = "mosquitto";
version = "0.0.1";
description = "An open source MQTT broker";
defaultProject = "mosquitto";
}

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
persistentVolumeClaims."${values.uniqueName}-data".spec = {
accessModes = ["ReadWriteOnce"];

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
services."${values.uniqueName}" = {
spec = {

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
statefulSets."${values.uniqueName}".spec = {
replicas = values.replicaCount;

View file

@ -1,17 +1,19 @@
{
lib,
utils,
project,
nixlet,
...
}:
with lib; {
with lib;
with utils;
with nixlet; {
# for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml
options = {
replicaCount = mkOption {
type = types.int;
default = 1;
};
image = utils.mkNestedOption {
image = mkNestedOption {
repository = mkOption {
type = types.str;
default = "eclipse-mosquitto";
@ -25,7 +27,7 @@ with lib; {
default = "IfNotPresent";
};
};
service = utils.mkNestedOption {
service = mkNestedOption {
port = mkOption {
type = types.int;
default = 1883;

View file

@ -0,0 +1,6 @@
{
name = "postgres";
version = "0.0.1";
description = "The world's most advanced open source database";
defaultProject = "postgres";
}

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
persistentVolumeClaims."${values.uniqueName}-data".spec = {
accessModes = ["ReadWriteOnce"];

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
services."${values.uniqueName}" = {
spec = {

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
statefulSets."${values.uniqueName}".spec = {
replicas = values.replicaCount;

View file

@ -1,17 +1,19 @@
{
lib,
utils,
project,
nixlet,
...
}:
with lib; {
with lib;
with utils;
with nixlet; {
# for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml
options = {
replicaCount = mkOption {
type = types.int;
default = 1;
};
image = utils.mkNestedOption {
image = mkNestedOption {
repository = mkOption {
type = types.str;
default = "postgres";
@ -25,7 +27,7 @@ with lib; {
default = "IfNotPresent";
};
};
service = utils.mkNestedOption {
service = mkNestedOption {
port = mkOption {
type = types.int;
default = 5432;

View file

@ -1,8 +1,9 @@
{
values,
nixlet,
lib,
...
}: {
}:
with nixlet; {
kubernetes.resources = {
deployments."${values.uniqueName}" = {
spec = {

View file

@ -0,0 +1,6 @@
{
name = "surrealdb";
version = "0.0.1";
description = "As a multi-model database, SurrealDB enables developers to use multiple techniques to store and model data, without having to choose a method in advance";
defaultProject = "surrealdb";
}

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
services."${values.uniqueName}" = {
spec = {

View file

@ -1,17 +1,19 @@
{
lib,
utils,
project,
nixlet,
...
}:
with lib; {
with lib;
with utils;
with nixlet; {
# for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml
options = {
replicaCount = mkOption {
type = types.int;
default = 1;
};
image = utils.mkNestedOption {
image = mkNestedOption {
repository = mkOption {
type = types.str;
default = "surrealdb/surrealdb";
@ -25,7 +27,7 @@ with lib; {
default = "latest";
};
};
service = utils.mkNestedOption {
service = mkNestedOption {
port = mkOption {
type = types.int;
default = 8000;
@ -35,7 +37,7 @@ with lib; {
default = "ClusterIP";
};
};
surrealdb = utils.mkNestedOption {
surrealdb = mkNestedOption {
log = mkOption {
type = types.str;
default = "info";

View file

@ -1,8 +1,9 @@
{
values,
nixlet,
pkgs,
...
}: let
}:
with nixlet; let
tomlFormat = pkgs.formats.toml {};
in {
kubernetes.resources = {

6
nixlets/tikv/nixlet.nix Normal file
View file

@ -0,0 +1,6 @@
{
name = "tikv";
version = "0.0.1";
description = "TiKV is an open-source, distributed, and transactional key-value database";
defaultProject = "tikv";
}

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
services = {
/*

View file

@ -1,4 +1,5 @@
{values, ...}: {
{nixlet, ...}:
with nixlet; {
kubernetes.resources = {
/*
Placement Driver

View file

@ -1,10 +1,12 @@
{
lib,
utils,
project,
nixlet,
...
}:
with lib; {
with lib;
with utils;
with nixlet; {
# for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml
options = {
pd = utils.mkNestedOption {

View file

@ -1,4 +1,4 @@
{values, ...}: {
{nixlet, ...}: {
imports = [
# import other files here
];

6
template/nixlet.nix Normal file
View file

@ -0,0 +1,6 @@
{
name = "";
version = "0.0.1";
description = "";
defaultProject = "";
}

View file

@ -1,9 +1,12 @@
{
lib,
utils,
nixlet,
...
}:
with lib; {
with lib;
with utils;
with nixlet; {
# for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml
options = {
# define values here