mirror of
https://github.com/TECHNOFAB11/kubenix.git
synced 2025-12-12 08:00:06 +01:00
fixups
This commit is contained in:
David Arnold
parent
bbc5e3d477
commit
1bd3fe4d4e
22 changed files with 49 additions and 609 deletions
|
|
@ -1,406 +0,0 @@
|
|||
{ nixosPath, config, pkgs, lib, kubenix, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.testing;
|
||||
k3s = pkgs.k3s;
|
||||
k3sAirgapImages = pkgs.callPackage ./k3s-airgap-images.nix {};
|
||||
toJSONFile = content: builtins.toFile "json" (builtins.toJSON content);
|
||||
|
||||
nixosTesting = import "${nixosPath}/lib/testing-python.nix" {
|
||||
inherit pkgs;
|
||||
system = "x86_64-linux";
|
||||
};
|
||||
|
||||
kubernetesBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
|
||||
master = findFirst
|
||||
(node: any (role: role == "master") node.config.services.kubernetes.roles)
|
||||
(throw "no master node")
|
||||
(attrValues nodes);
|
||||
extraHosts = ''
|
||||
${master.config.networking.primaryIPAddress} etcd.${config.networking.domain}
|
||||
${master.config.networking.primaryIPAddress} api.${config.networking.domain}
|
||||
${concatMapStringsSep "\n"
|
||||
(node: let n = node.config.networking; in "${n.primaryIPAddress} ${n.hostName}.${n.domain}")
|
||||
(attrValues nodes)}
|
||||
'';
|
||||
in {
|
||||
imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
|
||||
|
||||
config = mkMerge [{
|
||||
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
|
||||
virtualisation.memorySize = mkDefault 2048;
|
||||
virtualisation.cores = mkDefault 16;
|
||||
virtualisation.diskSize = mkDefault 4096;
|
||||
networking = {
|
||||
inherit extraHosts;
|
||||
domain = "my.xzy";
|
||||
nameservers = ["10.0.0.254"];
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
trustedInterfaces = ["docker0" "cni0"];
|
||||
|
||||
extraCommands = concatMapStrings (node: ''
|
||||
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
|
||||
'') (attrValues nodes);
|
||||
};
|
||||
};
|
||||
environment.systemPackages = [ pkgs.kubectl ];
|
||||
environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
|
||||
services.flannel.iface = "eth1";
|
||||
services.kubernetes = {
|
||||
easyCerts = true;
|
||||
apiserver = {
|
||||
securePort = 443;
|
||||
advertiseAddress = master.config.networking.primaryIPAddress;
|
||||
};
|
||||
masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
|
||||
};
|
||||
|
||||
systemd.extraConfig = "DefaultLimitNOFILE=1048576";
|
||||
|
||||
systemd.services.copy-certs = {
|
||||
description = "Share k8s certificates with host";
|
||||
script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
|
||||
after = [ "kubernetes.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
(mkIf (any (role: role == "master") config.services.kubernetes.roles) {
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
})];
|
||||
};
|
||||
|
||||
mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
|
||||
nixosTesting.makeTest {
|
||||
inherit name;
|
||||
|
||||
nodes.kube = { config, pkgs, nodes, ... }: {
|
||||
imports = [ kubernetesBaseConfig extraConfiguration ];
|
||||
services.kubernetes = {
|
||||
roles = ["master" "node"];
|
||||
flannel.enable = false;
|
||||
kubelet = {
|
||||
networkPlugin = "cni";
|
||||
cni.config = [{
|
||||
name = "mynet";
|
||||
type = "bridge";
|
||||
bridge = "cni0";
|
||||
addIf = true;
|
||||
ipMasq = true;
|
||||
isGateway = true;
|
||||
ipam = {
|
||||
type = "host-local";
|
||||
subnet = "10.1.0.0/16";
|
||||
gateway = "10.1.0.1";
|
||||
routes = [{
|
||||
dst = "0.0.0.0/0";
|
||||
}];
|
||||
};
|
||||
}];
|
||||
};
|
||||
};
|
||||
networking.primaryIPAddress = mkForce "192.168.1.1";
|
||||
};
|
||||
|
||||
skipLint = true;
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
kube.wait_until_succeeds("kubectl get node kube.my.xzy | grep -w Ready")
|
||||
|
||||
${testScript}
|
||||
'';
|
||||
};
|
||||
|
||||
k3sBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
|
||||
extraHosts = ''
|
||||
${concatMapStringsSep "\n"
|
||||
(node: let n = node.config.networking; in "${n.primaryIPAddress} ${n.hostName}.${n.domain}")
|
||||
(attrValues nodes)}
|
||||
'';
|
||||
in {
|
||||
imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
|
||||
|
||||
config = mkMerge [{
|
||||
virtualisation.memorySize = mkDefault 2048;
|
||||
virtualisation.cores = mkDefault 16;
|
||||
virtualisation.diskSize = mkDefault 4096;
|
||||
virtualisation.docker.enable = true;
|
||||
networking = {
|
||||
inherit extraHosts;
|
||||
domain = "my.xzy";
|
||||
nameservers = ["10.43.0.10"];
|
||||
firewall = {
|
||||
trustedInterfaces = ["docker0" "cni0"];
|
||||
|
||||
extraCommands = concatMapStrings (node: ''
|
||||
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
|
||||
'') (attrValues nodes);
|
||||
};
|
||||
};
|
||||
environment.systemPackages = [ pkgs.kubectl pkgs.docker k3s ];
|
||||
environment.variables.KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
|
||||
systemd.extraConfig = "DefaultLimitNOFILE=1048576";
|
||||
systemd.services.seed-docker-images = {
|
||||
description = "Copy k3s airgap images, and services.kubernetes.kubelet.seedDockerImages";
|
||||
wantedBy = ["k3s.service"];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = [
|
||||
"${pkgs.docker}/bin/docker load --input='${k3sAirgapImages}'"
|
||||
] ++ builtins.map
|
||||
(image : "${pkgs.docker}/bin/docker load --input='${image}'")
|
||||
config.services.kubernetes.kubelet.seedDockerImages;
|
||||
};
|
||||
};
|
||||
systemd.services.k3s = {
|
||||
description = "Lightweight Kubernetes";
|
||||
wantedBy = ["multi-user.target"];
|
||||
after = ["network-online.target" "seed-docker-images.service"];
|
||||
|
||||
serviceConfig = {
|
||||
# setting --service-cidr 10.0.0.0/24 --cluster-dns 10.0.0.254
|
||||
# was flakey, removed and hard coding nameservers to default value
|
||||
ExecStart = "${k3s}/bin/k3s server --docker";
|
||||
Type = "notify";
|
||||
KillMode = "process";
|
||||
Delegate = "yes";
|
||||
LimitNOFILE = "infinity";
|
||||
LimitNPROC = "infinity";
|
||||
LimitCORE = "infinity";
|
||||
TasksMax = "infinity";
|
||||
TimeoutStartSec = "0";
|
||||
Restart = "always";
|
||||
RestartSec = "5s";
|
||||
};
|
||||
};
|
||||
}
|
||||
(mkIf (any (role: role == "master") config.services.kubernetes.roles) {
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
})];
|
||||
};
|
||||
|
||||
mkK3sTest = { name, testScript, extraConfiguration ? {} }:
|
||||
|
||||
nixosTesting.makeTest {
|
||||
inherit name;
|
||||
|
||||
nodes.kube = { config, pkgs, nodes, ... }: {
|
||||
imports = [ k3sBaseConfig extraConfiguration ];
|
||||
networking.primaryIPAddress = mkForce "192.168.1.1";
|
||||
};
|
||||
|
||||
skipLint = true;
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
kube.wait_for_unit('k3s.service')
|
||||
kube.wait_until_succeeds("kubectl get node kube | grep -w Ready")
|
||||
|
||||
${testScript}
|
||||
'';
|
||||
};
|
||||
testOptions = { config, ... }: let
|
||||
modules = [config.module ./test.nix ./base.nix {
|
||||
config = {
|
||||
kubenix.project = mkDefault config.name;
|
||||
_module.args = {
|
||||
test = config;
|
||||
} // cfg.args;
|
||||
};
|
||||
}] ++ cfg.defaults;
|
||||
|
||||
test = (kubenix.evalModules {
|
||||
check = false;
|
||||
inherit modules;
|
||||
}).config.test;
|
||||
|
||||
evaled' = kubenix.evalModules {
|
||||
inherit modules;
|
||||
};
|
||||
|
||||
evaled =
|
||||
if cfg.throwError then evaled'
|
||||
else if (builtins.tryEval evaled'.config.test.assertions).success then evaled' else null;
|
||||
in {
|
||||
options = {
|
||||
name = mkOption {
|
||||
description = "test name";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
description = mkOption {
|
||||
description = "test description";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
enable = mkOption {
|
||||
description = "Whether to enable test";
|
||||
type = types.bool;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
module = mkOption {
|
||||
description = "Module defining submodule";
|
||||
type = types.unspecified;
|
||||
};
|
||||
|
||||
evaled = mkOption {
|
||||
description = "Test evaulation result";
|
||||
type = types.nullOr types.attrs;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
success = mkOption {
|
||||
description = "Whether test was success";
|
||||
type = types.bool;
|
||||
internal = true;
|
||||
default = false;
|
||||
};
|
||||
|
||||
assertions = mkOption {
|
||||
description = "Test result";
|
||||
type = types.unspecified;
|
||||
internal = true;
|
||||
default = [];
|
||||
};
|
||||
|
||||
test = mkOption {
|
||||
description = "Test derivation to run";
|
||||
type = types.nullOr types.package;
|
||||
default = null;
|
||||
};
|
||||
|
||||
result = mkOption {
|
||||
description = "Test result";
|
||||
type = types.package;
|
||||
};
|
||||
};
|
||||
|
||||
config = mkMerge [{
|
||||
inherit evaled;
|
||||
inherit (test) name description enable;
|
||||
result = pkgs.runCommand "${cfg.name}-${config.name}-test.json" {
|
||||
buildInputs = [ pkgs.jshon pkgs.jq ];
|
||||
} ''
|
||||
jshon -n object \
|
||||
-s "${config.name}" -i name \
|
||||
-s "${config.description}" -i description \
|
||||
-n "${if config.success then "true" else "false"}" -i success \
|
||||
${if config.test == null then "-n null" else "-s '${config.test}'"} -i test > result.json
|
||||
|
||||
jq -s '.[0].assertions = .[1] | .[0]' result.json ${toJSONFile (map (getAttrs ["assertion" "message"]) config.assertions)} > $out
|
||||
'';
|
||||
} (mkIf (config.evaled != null) {
|
||||
inherit (evaled.config.test) assertions;
|
||||
success = all (el: el.assertion) config.assertions;
|
||||
test =
|
||||
if cfg.e2e && evaled.config.test.testScript != null
|
||||
then
|
||||
if evaled.config.test.distro == null ||
|
||||
evaled.config.test.distro == "nixos" then
|
||||
mkKubernetesSingleNodeTest {
|
||||
name = config.name;
|
||||
inherit (evaled.config.test) testScript extraConfiguration;
|
||||
}
|
||||
else if evaled.config.test.distro == "k3s" then
|
||||
mkK3sTest {
|
||||
name = config.name;
|
||||
inherit (evaled.config.test) testScript extraConfiguration;
|
||||
}
|
||||
else throw "invalid test.distro ${evaled.config.test.distro}"
|
||||
else null;
|
||||
})];
|
||||
};
|
||||
in {
|
||||
options = {
|
||||
testing.name = mkOption {
|
||||
description = "Testing suite name";
|
||||
type = types.str;
|
||||
default = "default";
|
||||
};
|
||||
|
||||
testing.throwError = mkOption {
|
||||
description = "Whether to throw error";
|
||||
type = types.bool;
|
||||
default = true;
|
||||
};
|
||||
|
||||
testing.e2e = mkOption {
|
||||
description = "Whether to enable e2e tests";
|
||||
type = types.bool;
|
||||
default = true;
|
||||
};
|
||||
|
||||
testing.defaults = mkOption {
|
||||
description = "Testing defaults";
|
||||
type = types.coercedTo types.unspecified (value: [value]) (types.listOf types.unspecified);
|
||||
example = literalExample ''{config, ...}: {
|
||||
kubernetes.version = config.kubernetes.version;
|
||||
}'';
|
||||
apply = filter (v: v!=[]);
|
||||
default = [];
|
||||
};
|
||||
|
||||
testing.tests = mkOption {
|
||||
description = "List of test cases";
|
||||
default = [];
|
||||
type = types.listOf (types.coercedTo types.path (module: {inherit module;}) (types.submodule testOptions));
|
||||
apply = tests: filter (test: test.enable) tests;
|
||||
};
|
||||
|
||||
testing.args = mkOption {
|
||||
description = "Attribute set of extra args passed to tests";
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
};
|
||||
|
||||
testing.testsByName = mkOption {
|
||||
description = "Tests by name";
|
||||
type = types.attrsOf types.attrs;
|
||||
default = listToAttrs (map (test: nameValuePair test.name test) cfg.tests);
|
||||
};
|
||||
|
||||
testing.success = mkOption {
|
||||
description = "Whether testing was a success";
|
||||
type = types.bool;
|
||||
default = all (test: test.success) cfg.tests;
|
||||
};
|
||||
|
||||
testing.results = mkOption {
|
||||
description = "Test results";
|
||||
type = types.attrsOf types.package;
|
||||
default = mapAttrs (_: t: t.result) cfg.testsByName;
|
||||
};
|
||||
|
||||
testing.result = mkOption {
|
||||
description = "Testing results";
|
||||
type = types.package;
|
||||
default = pkgs.runCommand "${cfg.name}-test-results.json" {
|
||||
buildInputs = [ pkgs.jq ];
|
||||
} ''
|
||||
jq -s -r '.' ${concatMapStringsSep " " (t: t.result) cfg.tests} > tests.json
|
||||
jq -n \
|
||||
--rawfile tests tests.json \
|
||||
--argjson success `jq -s -r 'if all(.success) == true then true else false end' ${concatMapStringsSep " " (t: t.result) cfg.tests}` \
|
||||
'{"success": $success, "tests": $tests | fromjson }' > $out
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ let
|
|||
cfg = config.testing;
|
||||
|
||||
testModule = {
|
||||
imports = [ ./test.nix ];
|
||||
imports = [ ./evalTest.nix ];
|
||||
|
||||
# passthru testing configuration
|
||||
config._module.args = {
|
||||
|
|
@ -86,12 +86,14 @@ in {
|
|||
};
|
||||
|
||||
success = mkOption {
|
||||
internal = true; # read only property
|
||||
description = "Whether testing was a success";
|
||||
type = types.bool;
|
||||
default = all (test: test.success) cfg.tests;
|
||||
};
|
||||
|
||||
testScript = mkOption {
|
||||
internal = true; # set by test driver
|
||||
type = types.package;
|
||||
description = "Script to run e2e tests";
|
||||
};
|
||||
|
|
|
|||
|
|
@ -18,9 +18,9 @@ let
|
|||
${cfg.defaultHeader}
|
||||
${t.script}
|
||||
''
|
||||
else p.script;
|
||||
else t.script;
|
||||
|
||||
tests = pkgs.linkFarm "${testing.name}-tests" (map (t: {
|
||||
tests = builtins.trace testing pkgs.linkFarm "${testing.name}-tests" (map (t: {
|
||||
path = toTestScript t;
|
||||
name = "${t.name}_test.py";
|
||||
}) testing.tests);
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
{ lib, config, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.kubetest;
|
||||
|
||||
in {
|
||||
options.test.kubetest = {
|
||||
enable = mkOption {
|
||||
description = "Whether to use kubetest test driver";
|
||||
type = types.bool;
|
||||
default = cfg.testScript != "";
|
||||
};
|
||||
|
||||
testScript = mkOption {
|
||||
type = types.lines;
|
||||
description = "Test script to use for kubetest";
|
||||
default = "";
|
||||
};
|
||||
|
||||
extraPackages = mkOption {
|
||||
type = types.listOf types.package;
|
||||
description = "List of extra packages to use for kubetest";
|
||||
default = [];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
{ lib, config, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
{
|
||||
options.testing.kubetest = {
|
||||
defaultHeader = mkOption {
|
||||
description = "Default test header";
|
||||
type = types.lines;
|
||||
default = ''
|
||||
import pytest
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -55,29 +55,7 @@ let
|
|||
then evaled' else null;
|
||||
|
||||
in {
|
||||
imports = [
|
||||
./driver/kubetest.nix
|
||||
];
|
||||
|
||||
options = {
|
||||
name = mkOption {
|
||||
description = "test name";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
description = mkOption {
|
||||
description = "test description";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
enable = mkOption {
|
||||
description = "Whether to enable test";
|
||||
type = types.bool;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
module = mkOption {
|
||||
description = "Module defining kubenix test";
|
||||
type = types.unspecified;
|
||||
|
|
@ -96,6 +74,25 @@ in {
|
|||
default = false;
|
||||
};
|
||||
|
||||
# transparently forwarded from the test's `test` attribute for ease of access
|
||||
name = mkOption {
|
||||
description = "test name";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
description = mkOption {
|
||||
description = "test description";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
enable = mkOption {
|
||||
description = "Whether to enable test";
|
||||
type = types.bool;
|
||||
internal = true;
|
||||
};
|
||||
|
||||
assertions = mkOption {
|
||||
description = "Test result";
|
||||
type = types.unspecified;
|
||||
|
|
@ -109,26 +106,20 @@ in {
|
|||
internal = true;
|
||||
};
|
||||
|
||||
driver = mkOption {
|
||||
description = "Name of the driver to use for testing";
|
||||
type = types.str;
|
||||
internal = true;
|
||||
};
|
||||
};
|
||||
|
||||
config = mkMerge [
|
||||
{
|
||||
inherit evaled;
|
||||
inherit (testConfig) name description enable driver;
|
||||
inherit (testConfig) name description enable;
|
||||
}
|
||||
|
||||
# if test is evaled check assertions
|
||||
(mkIf (config.evaled != null) {
|
||||
inherit (evaled.config.test) assertions;
|
||||
inherit (evaled.config.test) assertions script;
|
||||
|
||||
# if all assertions are true, test is successfull
|
||||
success = all (el: el.assertion) config.assertions;
|
||||
script = evaled.config.test.script;
|
||||
})
|
||||
];
|
||||
}
|
||||
|
|
@ -61,6 +61,17 @@ let
|
|||
};
|
||||
|
||||
systemd.extraConfig = "DefaultLimitNOFILE=1048576";
|
||||
|
||||
systemd.services.copy-certs = {
|
||||
description = "Share k8s certificates with host";
|
||||
script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
|
||||
after = [ "kubernetes.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
# configuration only applied on master nodes
|
||||
|
|
|
|||
|
|
@ -52,21 +52,5 @@ in {
|
|||
type = types.nullOr (types.either types.lines types.path);
|
||||
};
|
||||
|
||||
testScript = mkOption {
|
||||
description = "Script to run as part of testing";
|
||||
type = types.nullOr types.lines;
|
||||
default = null;
|
||||
};
|
||||
|
||||
distro = mkOption {
|
||||
description = "Kubernetes distro to run the test with. Defaults to 'nixos', other option is 'k3s'";
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
driver = mkOption {
|
||||
description = "Name of the driver to use for testing";
|
||||
type = types.str;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue