mirror of
https://github.com/TECHNOFAB11/kubenix.git
synced 2025-12-12 08:00:06 +01:00
feat: refactor testing
This commit is contained in:
Jaka Hudoklin
parent
937ce15748
commit
9bc2406ff2
7 changed files with 158 additions and 163 deletions
|
|
@ -11,109 +11,101 @@ let
|
|||
system = "x86_64-linux";
|
||||
};
|
||||
|
||||
mkKubernetesBaseTest =
|
||||
{ name, domain ? "my.zyx", test, machines
|
||||
, extraConfiguration ? null }:
|
||||
let
|
||||
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
|
||||
master = machines.${masterName};
|
||||
extraHosts = ''
|
||||
${master.ip} etcd.${domain}
|
||||
${master.ip} api.${domain}
|
||||
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
|
||||
'';
|
||||
kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
|
||||
mkdir -p $out/bin
|
||||
makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
|
||||
'';
|
||||
in nixosTesting.makeTest {
|
||||
kubernetesBaseConfig = { config, pkgs, lib, nodes, ... }: let
|
||||
master = findFirst
|
||||
(node: any (role: role == "master") node.config.services.kubernetes.roles)
|
||||
(throw "no master node")
|
||||
(attrValues nodes);
|
||||
extraHosts = ''
|
||||
${master.config.networking.primaryIPAddress} etcd.${config.networking.domain}
|
||||
${master.config.networking.primaryIPAddress} api.${config.networking.domain}
|
||||
${concatMapStringsSep "\n"
|
||||
(node: let n = node.config.networking; in "${n.primaryIPAddress} ${n.hostName}.${n.domain}")
|
||||
(attrValues nodes)}
|
||||
'';
|
||||
in {
|
||||
imports = [ <nixpkgs/nixos/modules/profiles/minimal.nix> ];
|
||||
|
||||
config = mkMerge [{
|
||||
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
|
||||
virtualisation.memorySize = mkDefault 2048;
|
||||
virtualisation.cores = mkDefault "all";
|
||||
virtualisation.diskSize = mkDefault 4096;
|
||||
networking = {
|
||||
inherit extraHosts;
|
||||
domain = "my.xzy";
|
||||
nameservers = ["10.0.0.254"];
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
trustedInterfaces = ["docker0" "cni0"];
|
||||
|
||||
extraCommands = concatMapStrings (node: ''
|
||||
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
|
||||
'') (attrValues nodes);
|
||||
};
|
||||
};
|
||||
environment.systemPackages = [ pkgs.kubectl ];
|
||||
environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
|
||||
services.flannel.iface = "eth1";
|
||||
services.kubernetes = {
|
||||
easyCerts = true;
|
||||
apiserver = {
|
||||
securePort = 443;
|
||||
advertiseAddress = master.config.networking.primaryIPAddress;
|
||||
};
|
||||
masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
|
||||
};
|
||||
}
|
||||
(mkIf (any (role: role == "master") config.services.kubernetes.roles) {
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
})];
|
||||
};
|
||||
|
||||
mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
|
||||
nixosTesting.makeTest {
|
||||
inherit name;
|
||||
|
||||
nodes = mapAttrs (machineName: machine: { config, pkgs, lib, nodes, ... }: {
|
||||
imports = [<nixpkgs/nixos/modules/profiles/minimal.nix>];
|
||||
|
||||
config = mkMerge [{
|
||||
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
|
||||
virtualisation.memorySize = mkDefault 2048;
|
||||
virtualisation.cores = mkDefault "all";
|
||||
virtualisation.diskSize = mkDefault 4096;
|
||||
networking = {
|
||||
inherit domain extraHosts;
|
||||
nameservers = ["10.0.0.254"];
|
||||
primaryIPAddress = mkForce machine.ip;
|
||||
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
10250 # kubelet
|
||||
];
|
||||
trustedInterfaces = ["docker0" "cni0"];
|
||||
|
||||
extraCommands = concatMapStrings (node: ''
|
||||
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
|
||||
'') (attrValues nodes);
|
||||
};
|
||||
nodes.kube = { config, pkgs, lib, nodes, ... }: {
|
||||
imports = [ kubernetesBaseConfig ];
|
||||
services.kubernetes = {
|
||||
roles = ["master" "node"];
|
||||
flannel.enable = false;
|
||||
kubelet = {
|
||||
networkPlugin = "cni";
|
||||
cni.config = [{
|
||||
name = "mynet";
|
||||
type = "bridge";
|
||||
bridge = "cni0";
|
||||
addIf = true;
|
||||
ipMasq = true;
|
||||
isGateway = true;
|
||||
ipam = {
|
||||
type = "host-local";
|
||||
subnet = "10.1.0.0/16";
|
||||
gateway = "10.1.0.1";
|
||||
routes = [{
|
||||
dst = "0.0.0.0/0";
|
||||
}];
|
||||
};
|
||||
}];
|
||||
};
|
||||
environment.systemPackages = [ kubectl ];
|
||||
services.flannel.iface = "eth1";
|
||||
services.kubernetes = {
|
||||
easyCerts = true;
|
||||
inherit (machine) roles;
|
||||
apiserver = {
|
||||
securePort = 443;
|
||||
advertiseAddress = master.ip;
|
||||
};
|
||||
masterAddress = "${masterName}.${config.networking.domain}";
|
||||
};
|
||||
}
|
||||
(optionalAttrs (any (role: role == "master") machine.roles) {
|
||||
networking.firewall.allowedTCPPorts = [
|
||||
443 # kubernetes apiserver
|
||||
];
|
||||
})
|
||||
(optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
|
||||
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))];
|
||||
}) machines;
|
||||
};
|
||||
networking.primaryIPAddress = mkForce "192.168.1.1";
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
startAll;
|
||||
|
||||
${test}
|
||||
$kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
|
||||
|
||||
${testScript}
|
||||
'';
|
||||
};
|
||||
|
||||
mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
|
||||
machines = {
|
||||
kube = {
|
||||
roles = ["master" "node"];
|
||||
ip = "192.168.1.1";
|
||||
};
|
||||
};
|
||||
extraConfiguration = {...}: {
|
||||
services.kubernetes.flannel.enable = false;
|
||||
services.kubernetes.kubelet = {
|
||||
networkPlugin = "cni";
|
||||
cni.config = [{
|
||||
name = "mynet";
|
||||
type = "bridge";
|
||||
bridge = "cni0";
|
||||
addIf = true;
|
||||
ipMasq = true;
|
||||
isGateway = true;
|
||||
ipam = {
|
||||
type = "host-local";
|
||||
subnet = "10.1.0.0/16";
|
||||
gateway = "10.1.0.1";
|
||||
routes = [{
|
||||
dst = "0.0.0.0/0";
|
||||
}];
|
||||
};
|
||||
}];
|
||||
};
|
||||
};
|
||||
} // attrs // {
|
||||
name = "kubernetes-${attrs.name}-singlenode";
|
||||
});
|
||||
|
||||
testOptions = {config, ...}: let
|
||||
modules = [config.module ./test.nix {
|
||||
config._module.args.test = config;
|
||||
|
|
@ -179,17 +171,12 @@ let
|
|||
default = [];
|
||||
};
|
||||
|
||||
script = mkOption {
|
||||
description = "Test script";
|
||||
test = mkOption {
|
||||
description = "Test derivation to run";
|
||||
type = types.nullOr types.package;
|
||||
default = null;
|
||||
};
|
||||
|
||||
driver = mkOption {
|
||||
description = "Testing driver";
|
||||
type = types.nullOr types.package;
|
||||
};
|
||||
|
||||
generated = mkOption {
|
||||
description = "Generated resources";
|
||||
type = types.nullOr types.package;
|
||||
|
|
@ -202,18 +189,14 @@ let
|
|||
} (mkIf config.evaled {
|
||||
inherit (evaled.config.test) assertions;
|
||||
success = all (el: el.assertion) config.assertions;
|
||||
script =
|
||||
if cfg.e2e && evaled.config.test.check != null
|
||||
test =
|
||||
if cfg.e2e && evaled.config.test.testScript != null
|
||||
then mkKubernetesSingleNodeTest {
|
||||
inherit (evaled.config.test) testScript;
|
||||
name = config.name;
|
||||
test = ''
|
||||
$kube->waitUntilSucceeds("kubectl get node kube.my.zyx | grep -w Ready");
|
||||
${evaled.config.test.check}
|
||||
'';
|
||||
} else null;
|
||||
driver = mkIf (config.script != null) config.script.driver;
|
||||
generated = mkIf (hasAttr "kubernetes" evaled.config)
|
||||
pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated);
|
||||
(pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated));
|
||||
})];
|
||||
};
|
||||
in {
|
||||
|
|
@ -264,9 +247,8 @@ in {
|
|||
default = pkgs.writeText "testing-report.json" (builtins.toJSON {
|
||||
success = cfg.success;
|
||||
tests = map (test: {
|
||||
inherit (test) name description evaled success;
|
||||
inherit (test) name description evaled success test;
|
||||
assertions = moduleToAttrs test.assertions;
|
||||
script = test.script;
|
||||
}) (filter (test: test.enable) cfg.tests);
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -51,10 +51,16 @@ in {
|
|||
type = types.listOf types.package;
|
||||
};
|
||||
|
||||
check = mkOption {
|
||||
description = "Script to run as part testing";
|
||||
testScript = mkOption {
|
||||
description = "Script to run as part of testing";
|
||||
type = types.nullOr types.lines;
|
||||
default = null;
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
description = "Extra configuration for running test";
|
||||
type = types.unspecified;
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@
|
|||
with lib;
|
||||
|
||||
let
|
||||
images = pkgs.callPackage ./images.nix {};
|
||||
|
||||
tests = listToAttrs (map (version: let
|
||||
version' = replaceStrings ["."] ["_"] version;
|
||||
in nameValuePair "v${version'}" (evalModules {
|
||||
|
|
@ -28,11 +30,13 @@ let
|
|||
./k8s/deployment.nix
|
||||
./k8s/crd.nix
|
||||
./k8s/1.13/crd.nix
|
||||
./k8s/submodule.nix
|
||||
./submodules/simple.nix
|
||||
];
|
||||
testing.defaults = ({kubenix, ...}: {
|
||||
imports = [kubenix.k8s];
|
||||
kubernetes.version = version;
|
||||
_module.args.images = images;
|
||||
});
|
||||
}
|
||||
];
|
||||
|
|
@ -43,7 +47,4 @@ let
|
|||
inherit kubenix;
|
||||
};
|
||||
}).config) k8sVersions);
|
||||
in {
|
||||
inherit tests;
|
||||
results = mapAttrs (_: test: test.testing.result) tests;
|
||||
}
|
||||
in tests
|
||||
|
|
|
|||
45
tests/images.nix
Normal file
45
tests/images.nix
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
{ pkgs, dockerTools, lib, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
{
|
||||
nginx = let
|
||||
nginxPort = "80";
|
||||
nginxConf = pkgs.writeText "nginx.conf" ''
|
||||
user nginx nginx;
|
||||
daemon off;
|
||||
error_log /dev/stdout info;
|
||||
pid /dev/null;
|
||||
events {}
|
||||
http {
|
||||
access_log /dev/stdout;
|
||||
server {
|
||||
listen ${nginxPort};
|
||||
index index.html;
|
||||
location / {
|
||||
root ${nginxWebRoot};
|
||||
}
|
||||
}
|
||||
}
|
||||
'';
|
||||
nginxWebRoot = pkgs.writeTextDir "index.html" ''
|
||||
<html><body><h1>Hello from NGINX</h1></body></html>
|
||||
'';
|
||||
in dockerTools.buildLayeredImage {
|
||||
name = "xtruder/nginx";
|
||||
tag = "latest";
|
||||
contents = [pkgs.nginx];
|
||||
extraCommands = ''
|
||||
mkdir etc
|
||||
chmod u+w etc
|
||||
echo "nginx:x:1000:1000::/:" > etc/passwd
|
||||
echo "nginx:x:1000:nginx" > etc/group
|
||||
'';
|
||||
config = {
|
||||
Cmd = ["nginx" "-c" nginxConf];
|
||||
ExposedPorts = {
|
||||
"${nginxPort}/tcp" = {};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -17,7 +17,7 @@ in {
|
|||
message = "should have group set";
|
||||
assertion = cfg.spec.group == "stable.example.com";
|
||||
}];
|
||||
check = ''
|
||||
testScript = ''
|
||||
$kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
|
||||
$kube->succeed("kubectl get crds | grep -i crontabs");
|
||||
$kube->succeed("kubectl get crontabs | grep -i crontab");
|
||||
|
|
|
|||
|
|
@ -1,56 +1,17 @@
|
|||
{ config, lib, pkgs, test, kubenix, ... }:
|
||||
{ config, lib, pkgs, test, kubenix, images, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.kubernetes.api.deployments.nginx;
|
||||
|
||||
nginxImage = let
|
||||
nginxPort = "80";
|
||||
nginxConf = pkgs.writeText "nginx.conf" ''
|
||||
user nginx nginx;
|
||||
daemon off;
|
||||
error_log /dev/stdout info;
|
||||
pid /dev/null;
|
||||
events {}
|
||||
http {
|
||||
access_log /dev/stdout;
|
||||
server {
|
||||
listen ${nginxPort};
|
||||
index index.html;
|
||||
location / {
|
||||
root ${nginxWebRoot};
|
||||
}
|
||||
}
|
||||
}
|
||||
'';
|
||||
nginxWebRoot = pkgs.writeTextDir "index.html" ''
|
||||
<html><body><h1>Hello from NGINX</h1></body></html>
|
||||
'';
|
||||
in pkgs.dockerTools.buildLayeredImage {
|
||||
name = "xtruder/nginx";
|
||||
tag = "latest";
|
||||
contents = [pkgs.nginx];
|
||||
extraCommands = ''
|
||||
mkdir etc
|
||||
chmod u+w etc
|
||||
echo "nginx:x:1000:1000::/:" > etc/passwd
|
||||
echo "nginx:x:1000:nginx" > etc/group
|
||||
'';
|
||||
config = {
|
||||
Cmd = ["nginx" "-c" nginxConf];
|
||||
ExposedPorts = {
|
||||
"${nginxPort}/tcp" = {};
|
||||
};
|
||||
};
|
||||
};
|
||||
image = images.nginx;
|
||||
in {
|
||||
imports = [
|
||||
kubenix.k8s
|
||||
];
|
||||
|
||||
test = {
|
||||
name = "k8s-deployment-simple";
|
||||
name = "k8s-deployment";
|
||||
description = "Simple k8s testing a simple deployment";
|
||||
assertions = [{
|
||||
message = "should have correct apiVersion and kind set";
|
||||
|
|
@ -59,8 +20,8 @@ in {
|
|||
message = "should have replicas set";
|
||||
assertion = cfg.spec.replicas == 10;
|
||||
}];
|
||||
check = ''
|
||||
$kube->waitUntilSucceeds("docker load < ${nginxImage}");
|
||||
testScript = ''
|
||||
$kube->waitUntilSucceeds("docker load < ${image}");
|
||||
$kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
|
||||
|
||||
$kube->succeed("kubectl get deployment | grep -i nginx");
|
||||
|
|
@ -76,7 +37,7 @@ in {
|
|||
template.metadata.labels.app = "nginx";
|
||||
template.spec = {
|
||||
containers.nginx = {
|
||||
image = "xtruder/nginx:latest";
|
||||
image = "${image.imageName}:${image.imageTag}";
|
||||
imagePullPolicy = "Never";
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ in {
|
|||
];
|
||||
|
||||
test = {
|
||||
name = "submodules/simple";
|
||||
name = "submodules-simple";
|
||||
description = "Simple k8s submodule test";
|
||||
assertions = [{
|
||||
message = "Submodule name is set";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue