diff --git a/testing/default.nix b/testing/default.nix index a845ad6..1c1a431 100644 --- a/testing/default.nix +++ b/testing/default.nix @@ -11,109 +11,101 @@ let system = "x86_64-linux"; }; - mkKubernetesBaseTest = - { name, domain ? "my.zyx", test, machines - , extraConfiguration ? null }: - let - masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines)); - master = machines.${masterName}; - extraHosts = '' - ${master.ip} etcd.${domain} - ${master.ip} api.${domain} - ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)} - ''; - kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } '' - mkdir -p $out/bin - makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig" - ''; - in nixosTesting.makeTest { + kubernetesBaseConfig = { config, pkgs, lib, nodes, ... }: let + master = findFirst + (node: any (role: role == "master") node.config.services.kubernetes.roles) + (throw "no master node") + (attrValues nodes); + extraHosts = '' + ${master.config.networking.primaryIPAddress} etcd.${config.networking.domain} + ${master.config.networking.primaryIPAddress} api.${config.networking.domain} + ${concatMapStringsSep "\n" + (node: let n = node.config.networking; in "${n.primaryIPAddress} ${n.hostName}.${n.domain}") + (attrValues nodes)} + ''; + in { + imports = [ ]; + + config = mkMerge [{ + boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*"; + virtualisation.memorySize = mkDefault 2048; + virtualisation.cores = mkDefault "all"; + virtualisation.diskSize = mkDefault 4096; + networking = { + inherit extraHosts; + domain = "my.xzy"; + nameservers = ["10.0.0.254"]; + firewall = { + allowedTCPPorts = [ + 10250 # kubelet + ]; + trustedInterfaces = ["docker0" "cni0"]; + + extraCommands = concatMapStrings (node: '' + iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT + '') (attrValues nodes); + }; + }; + environment.systemPackages = [ pkgs.kubectl ]; + environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig"; + services.flannel.iface = "eth1"; + services.kubernetes = { + easyCerts = true; + apiserver = { + securePort = 443; + advertiseAddress = master.config.networking.primaryIPAddress; + }; + masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}"; + }; + } + (mkIf (any (role: role == "master") config.services.kubernetes.roles) { + networking.firewall.allowedTCPPorts = [ + 443 # kubernetes apiserver + ]; + })]; + }; + + mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }: + nixosTesting.makeTest { inherit name; - nodes = mapAttrs (machineName: machine: { config, pkgs, lib, nodes, ... }: { - imports = []; - - config = mkMerge [{ - boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*"; - virtualisation.memorySize = mkDefault 2048; - virtualisation.cores = mkDefault "all"; - virtualisation.diskSize = mkDefault 4096; - networking = { - inherit domain extraHosts; - nameservers = ["10.0.0.254"]; - primaryIPAddress = mkForce machine.ip; - - firewall = { - allowedTCPPorts = [ - 10250 # kubelet - ]; - trustedInterfaces = ["docker0" "cni0"]; - - extraCommands = concatMapStrings (node: '' - iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT - '') (attrValues nodes); - }; + nodes.kube = { config, pkgs, lib, nodes, ... }: { + imports = [ kubernetesBaseConfig ]; + services.kubernetes = { + roles = ["master" "node"]; + flannel.enable = false; + kubelet = { + networkPlugin = "cni"; + cni.config = [{ + name = "mynet"; + type = "bridge"; + bridge = "cni0"; + addIf = true; + ipMasq = true; + isGateway = true; + ipam = { + type = "host-local"; + subnet = "10.1.0.0/16"; + gateway = "10.1.0.1"; + routes = [{ + dst = "0.0.0.0/0"; + }]; + }; + }]; }; - environment.systemPackages = [ kubectl ]; - services.flannel.iface = "eth1"; - services.kubernetes = { - easyCerts = true; - inherit (machine) roles; - apiserver = { - securePort = 443; - advertiseAddress = master.ip; - }; - masterAddress = "${masterName}.${config.networking.domain}"; - }; - } - (optionalAttrs (any (role: role == "master") machine.roles) { - networking.firewall.allowedTCPPorts = [ - 443 # kubernetes apiserver - ]; - }) - (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; })) - (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))]; - }) machines; + }; + networking.primaryIPAddress = mkForce "192.168.1.1"; + }; testScript = '' startAll; - ${test} + $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready"); + + ${testScript} ''; }; - mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({ - machines = { - kube = { - roles = ["master" "node"]; - ip = "192.168.1.1"; - }; - }; - extraConfiguration = {...}: { - services.kubernetes.flannel.enable = false; - services.kubernetes.kubelet = { - networkPlugin = "cni"; - cni.config = [{ - name = "mynet"; - type = "bridge"; - bridge = "cni0"; - addIf = true; - ipMasq = true; - isGateway = true; - ipam = { - type = "host-local"; - subnet = "10.1.0.0/16"; - gateway = "10.1.0.1"; - routes = [{ - dst = "0.0.0.0/0"; - }]; - }; - }]; - }; - }; - } // attrs // { - name = "kubernetes-${attrs.name}-singlenode"; - }); - testOptions = {config, ...}: let modules = [config.module ./test.nix { config._module.args.test = config; @@ -179,17 +171,12 @@ let default = []; }; - script = mkOption { - description = "Test script"; + test = mkOption { + description = "Test derivation to run"; type = types.nullOr types.package; default = null; }; - driver = mkOption { - description = "Testing driver"; - type = types.nullOr types.package; - }; - generated = mkOption { description = "Generated resources"; type = types.nullOr types.package; @@ -202,18 +189,14 @@ let } (mkIf config.evaled { inherit (evaled.config.test) assertions; success = all (el: el.assertion) config.assertions; - script = - if cfg.e2e && evaled.config.test.check != null + test = + if cfg.e2e && evaled.config.test.testScript != null then mkKubernetesSingleNodeTest { + inherit (evaled.config.test) testScript; name = config.name; - test = '' - $kube->waitUntilSucceeds("kubectl get node kube.my.zyx | grep -w Ready"); - ${evaled.config.test.check} - ''; } else null; - driver = mkIf (config.script != null) config.script.driver; generated = mkIf (hasAttr "kubernetes" evaled.config) - pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated); + (pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated)); })]; }; in { @@ -264,9 +247,8 @@ in { default = pkgs.writeText "testing-report.json" (builtins.toJSON { success = cfg.success; tests = map (test: { - inherit (test) name description evaled success; + inherit (test) name description evaled success test; assertions = moduleToAttrs test.assertions; - script = test.script; }) (filter (test: test.enable) cfg.tests); }); }; diff --git a/testing/test.nix b/testing/test.nix index ce03781..07fd1a1 100644 --- a/testing/test.nix +++ b/testing/test.nix @@ -51,10 +51,16 @@ in { type = types.listOf types.package; }; - check = mkOption { - description = "Script to run as part testing"; + testScript = mkOption { + description = "Script to run as part of testing"; type = types.nullOr types.lines; default = null; }; + + extraConfig = mkOption { + description = "Extra configuration for running test"; + type = types.unspecified; + default = {}; + }; }; } diff --git a/tests/default.nix b/tests/default.nix index ffc1c37..c04577d 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -10,6 +10,8 @@ with lib; let + images = pkgs.callPackage ./images.nix {}; + tests = listToAttrs (map (version: let version' = replaceStrings ["."] ["_"] version; in nameValuePair "v${version'}" (evalModules { @@ -28,11 +30,13 @@ let ./k8s/deployment.nix ./k8s/crd.nix ./k8s/1.13/crd.nix + ./k8s/submodule.nix ./submodules/simple.nix ]; testing.defaults = ({kubenix, ...}: { imports = [kubenix.k8s]; kubernetes.version = version; + _module.args.images = images; }); } ]; @@ -43,7 +47,4 @@ let inherit kubenix; }; }).config) k8sVersions); -in { - inherit tests; - results = mapAttrs (_: test: test.testing.result) tests; -} +in tests diff --git a/tests/images.nix b/tests/images.nix new file mode 100644 index 0000000..ee6eb9a --- /dev/null +++ b/tests/images.nix @@ -0,0 +1,45 @@ +{ pkgs, dockerTools, lib, ... }: + +with lib; + +{ + nginx = let + nginxPort = "80"; + nginxConf = pkgs.writeText "nginx.conf" '' + user nginx nginx; + daemon off; + error_log /dev/stdout info; + pid /dev/null; + events {} + http { + access_log /dev/stdout; + server { + listen ${nginxPort}; + index index.html; + location / { + root ${nginxWebRoot}; + } + } + } + ''; + nginxWebRoot = pkgs.writeTextDir "index.html" '' +

Hello from NGINX

+ ''; + in dockerTools.buildLayeredImage { + name = "xtruder/nginx"; + tag = "latest"; + contents = [pkgs.nginx]; + extraCommands = '' + mkdir etc + chmod u+w etc + echo "nginx:x:1000:1000::/:" > etc/passwd + echo "nginx:x:1000:nginx" > etc/group + ''; + config = { + Cmd = ["nginx" "-c" nginxConf]; + ExposedPorts = { + "${nginxPort}/tcp" = {}; + }; + }; + }; +} diff --git a/tests/k8s/crd.nix b/tests/k8s/crd.nix index bb3e55c..11d4b51 100644 --- a/tests/k8s/crd.nix +++ b/tests/k8s/crd.nix @@ -17,7 +17,7 @@ in { message = "should have group set"; assertion = cfg.spec.group == "stable.example.com"; }]; - check = '' + testScript = '' $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}"); $kube->succeed("kubectl get crds | grep -i crontabs"); $kube->succeed("kubectl get crontabs | grep -i crontab"); diff --git a/tests/k8s/deployment.nix b/tests/k8s/deployment.nix index c2b32bc..e1b3919 100644 --- a/tests/k8s/deployment.nix +++ b/tests/k8s/deployment.nix @@ -1,56 +1,17 @@ -{ config, lib, pkgs, test, kubenix, ... }: +{ config, lib, pkgs, test, kubenix, images, ... }: with lib; let cfg = config.kubernetes.api.deployments.nginx; - - nginxImage = let - nginxPort = "80"; - nginxConf = pkgs.writeText "nginx.conf" '' - user nginx nginx; - daemon off; - error_log /dev/stdout info; - pid /dev/null; - events {} - http { - access_log /dev/stdout; - server { - listen ${nginxPort}; - index index.html; - location / { - root ${nginxWebRoot}; - } - } - } - ''; - nginxWebRoot = pkgs.writeTextDir "index.html" '' -

Hello from NGINX

- ''; - in pkgs.dockerTools.buildLayeredImage { - name = "xtruder/nginx"; - tag = "latest"; - contents = [pkgs.nginx]; - extraCommands = '' - mkdir etc - chmod u+w etc - echo "nginx:x:1000:1000::/:" > etc/passwd - echo "nginx:x:1000:nginx" > etc/group - ''; - config = { - Cmd = ["nginx" "-c" nginxConf]; - ExposedPorts = { - "${nginxPort}/tcp" = {}; - }; - }; - }; + image = images.nginx; in { imports = [ kubenix.k8s ]; test = { - name = "k8s-deployment-simple"; + name = "k8s-deployment"; description = "Simple k8s testing a simple deployment"; assertions = [{ message = "should have correct apiVersion and kind set"; @@ -59,8 +20,8 @@ in { message = "should have replicas set"; assertion = cfg.spec.replicas == 10; }]; - check = '' - $kube->waitUntilSucceeds("docker load < ${nginxImage}"); + testScript = '' + $kube->waitUntilSucceeds("docker load < ${image}"); $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}"); $kube->succeed("kubectl get deployment | grep -i nginx"); @@ -76,7 +37,7 @@ in { template.metadata.labels.app = "nginx"; template.spec = { containers.nginx = { - image = "xtruder/nginx:latest"; + image = "${image.imageName}:${image.imageTag}"; imagePullPolicy = "Never"; }; }; diff --git a/tests/submodules/simple.nix b/tests/submodules/simple.nix index 457411a..4d967d8 100644 --- a/tests/submodules/simple.nix +++ b/tests/submodules/simple.nix @@ -10,7 +10,7 @@ in { ]; test = { - name = "submodules/simple"; + name = "submodules-simple"; description = "Simple k8s submodule test"; assertions = [{ message = "Submodule name is set";