feat: refactor testing

testing/default.nix

@@ -11,109 +11,101 @@ let
     system = "x86_64-linux";
   };
 
-  mkKubernetesBaseTest =
+  kubernetesBaseConfig = { config, pkgs, lib, nodes, ... }: let
-    { name, domain ? "my.zyx", test, machines
+    master = findFirst
-    , extraConfiguration ? null }:
+      (node: any (role: role == "master") node.config.services.kubernetes.roles)
-    let
+      (throw "no master node")
-      masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
+      (attrValues nodes);
-      master = machines.${masterName};
+    extraHosts = ''
-      extraHosts = ''
+      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
-        ${master.ip}  etcd.${domain}
+      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
-        ${master.ip}  api.${domain}
+      ${concatMapStringsSep "\n"
-        ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
+        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
-      '';
+        (attrValues nodes)}
-      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
+    '';
-        mkdir -p $out/bin
+  in {
-        makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
+    imports = [ <nixpkgs/nixos/modules/profiles/minimal.nix> ];
-      '';
+
-    in nixosTesting.makeTest {
+    config = mkMerge [{
+      boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+      virtualisation.memorySize = mkDefault 2048;
+      virtualisation.cores = mkDefault "all";
+      virtualisation.diskSize = mkDefault 4096;
+      networking = {
+        inherit extraHosts;
+        domain = "my.xzy";
+        nameservers = ["10.0.0.254"];
+        firewall = {
+          allowedTCPPorts = [
+            10250 # kubelet
+          ];
+          trustedInterfaces = ["docker0" "cni0"];
+
+          extraCommands = concatMapStrings  (node: ''
+            iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+          '') (attrValues nodes);
+        };
+      };
+      environment.systemPackages = [ pkgs.kubectl ];
+      environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
+      services.flannel.iface = "eth1";
+      services.kubernetes = {
+        easyCerts = true;
+        apiserver = {
+          securePort = 443;
+          advertiseAddress = master.config.networking.primaryIPAddress;
+        };
+        masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
+      };
+    }
+    (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
+      networking.firewall.allowedTCPPorts = [
+        443 # kubernetes apiserver
+      ];
+    })];
+  };
+
+  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
+    nixosTesting.makeTest {
       inherit name;
 
-      nodes = mapAttrs (machineName: machine: { config, pkgs, lib, nodes, ... }: {
+      nodes.kube = { config, pkgs, lib, nodes, ... }: {
-        imports = [<nixpkgs/nixos/modules/profiles/minimal.nix>];
+        imports = [ kubernetesBaseConfig ];
-
+        services.kubernetes = {
-        config = mkMerge [{
+          roles = ["master" "node"];
-          boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+          flannel.enable = false;
-          virtualisation.memorySize = mkDefault 2048;
+          kubelet = {
-          virtualisation.cores = mkDefault "all";
+            networkPlugin = "cni";
-          virtualisation.diskSize = mkDefault 4096;
+            cni.config = [{
-          networking = {
+              name = "mynet";
-            inherit domain extraHosts;
+              type = "bridge";
-            nameservers = ["10.0.0.254"];
+              bridge = "cni0";
-            primaryIPAddress = mkForce machine.ip;
+              addIf = true;
-
+              ipMasq = true;
-            firewall = {
+              isGateway = true;
-              allowedTCPPorts = [
+              ipam = {
-                10250 # kubelet
+                type = "host-local";
-              ];
+                subnet = "10.1.0.0/16";
-              trustedInterfaces = ["docker0" "cni0"];
+                gateway = "10.1.0.1";
-
+                routes = [{
-              extraCommands = concatMapStrings  (node: ''
+                  dst = "0.0.0.0/0";
-                iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+                }];
-              '') (attrValues nodes);
+              };
-            };
+            }];
           };
-          environment.systemPackages = [ kubectl ];
+        };
-          services.flannel.iface = "eth1";
+        networking.primaryIPAddress = mkForce "192.168.1.1";
-          services.kubernetes = {
+      };
-            easyCerts = true;
-            inherit (machine) roles;
-            apiserver = {
-              securePort = 443;
-              advertiseAddress = master.ip;
-            };
-            masterAddress = "${masterName}.${config.networking.domain}";
-          };
-        }
-        (optionalAttrs (any (role: role == "master") machine.roles) {
-          networking.firewall.allowedTCPPorts = [
-            443 # kubernetes apiserver
-          ];
-        })
-        (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
-        (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))];
-      }) machines;
 
       testScript = ''
         startAll;
 
-        ${test}
+        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
+
+        ${testScript}
       '';
     };
 
-  mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
-    machines = {
-      kube = {
-        roles = ["master" "node"];
-        ip = "192.168.1.1";
-      };
-    };
-    extraConfiguration = {...}: {
-      services.kubernetes.flannel.enable = false;
-      services.kubernetes.kubelet = {
-        networkPlugin = "cni";
-        cni.config = [{
-          name = "mynet";
-          type = "bridge";
-          bridge = "cni0";
-          addIf = true;
-          ipMasq = true;
-          isGateway = true;
-          ipam = {
-            type = "host-local";
-            subnet = "10.1.0.0/16";
-            gateway = "10.1.0.1";
-            routes = [{
-              dst = "0.0.0.0/0";
-            }];
-          };
-        }];
-      };
-    };
-  } // attrs // {
-    name = "kubernetes-${attrs.name}-singlenode";
-  });
-
   testOptions = {config, ...}: let
     modules = [config.module ./test.nix {
       config._module.args.test = config;
@@ -179,17 +171,12 @@ let
         default = [];
       };
 
-      script = mkOption {
+      test = mkOption {
-        description = "Test script";
+        description = "Test derivation to run";
         type = types.nullOr types.package;
         default = null;
       };
 
-      driver = mkOption {
-        description = "Testing driver";
-        type = types.nullOr types.package;
-      };
-
       generated = mkOption {
         description = "Generated resources";
         type = types.nullOr types.package;
@@ -202,18 +189,14 @@ let
     } (mkIf config.evaled {
       inherit (evaled.config.test) assertions;
       success = all (el: el.assertion) config.assertions;
-      script =
+      test =
-        if cfg.e2e && evaled.config.test.check != null
+        if cfg.e2e && evaled.config.test.testScript != null
         then mkKubernetesSingleNodeTest {
+          inherit (evaled.config.test) testScript;
           name = config.name;
-          test = ''
-            $kube->waitUntilSucceeds("kubectl get node kube.my.zyx | grep -w Ready");
-            ${evaled.config.test.check}
-          '';
         } else null;
-      driver = mkIf (config.script != null) config.script.driver;
       generated = mkIf (hasAttr "kubernetes" evaled.config)
-        pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated);
+        (pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated));
     })];
   };
 in {
@@ -264,9 +247,8 @@ in {
       default = pkgs.writeText "testing-report.json" (builtins.toJSON {
         success = cfg.success;
         tests = map (test: {
-          inherit (test) name description evaled success;
+          inherit (test) name description evaled success test;
           assertions = moduleToAttrs test.assertions;
-          script = test.script;
         }) (filter (test: test.enable) cfg.tests);
       });
     };

testing/test.nix

@@ -51,10 +51,16 @@ in {
       type = types.listOf types.package;
     };
 
-    check = mkOption {
+    testScript = mkOption {
-      description = "Script to run as part testing";
+      description = "Script to run as part of testing";
       type = types.nullOr types.lines;
       default = null;
     };
+
+    extraConfig = mkOption {
+      description = "Extra configuration for running test";
+      type = types.unspecified;
+      default = {};
+    };
   };
 }

tests/default.nix

@@ -10,6 +10,8 @@
 with lib;
 
 let
+  images = pkgs.callPackage ./images.nix {};
+
   tests = listToAttrs (map (version: let
     version' = replaceStrings ["."] ["_"] version;
   in nameValuePair "v${version'}" (evalModules {
@@ -28,11 +30,13 @@ let
           ./k8s/deployment.nix
           ./k8s/crd.nix
           ./k8s/1.13/crd.nix
+          ./k8s/submodule.nix
           ./submodules/simple.nix
         ];
         testing.defaults = ({kubenix, ...}: {
           imports = [kubenix.k8s];
           kubernetes.version = version;
+          _module.args.images = images;
         });
       }
     ];
@@ -43,7 +47,4 @@ let
       inherit kubenix;
     };
   }).config) k8sVersions);
-in {
+in tests
-  inherit tests;
-  results = mapAttrs (_: test: test.testing.result) tests;
-}

tests/images.nix

@@ -0,0 +1,45 @@
+{ pkgs, dockerTools, lib, ... }:
+
+with lib;
+
+{
+  nginx = let
+    nginxPort = "80";
+    nginxConf = pkgs.writeText "nginx.conf" ''
+      user nginx nginx;
+      daemon off;
+      error_log /dev/stdout info;
+      pid /dev/null;
+      events {}
+      http {
+        access_log /dev/stdout;
+        server {
+          listen ${nginxPort};
+          index index.html;
+          location / {
+            root ${nginxWebRoot};
+          }
+        }
+      }
+    '';
+    nginxWebRoot = pkgs.writeTextDir "index.html" ''
+      <html><body><h1>Hello from NGINX</h1></body></html>
+    '';
+  in dockerTools.buildLayeredImage {
+    name = "xtruder/nginx";
+    tag = "latest";
+    contents = [pkgs.nginx];
+    extraCommands = ''
+      mkdir etc
+      chmod u+w etc
+      echo "nginx:x:1000:1000::/:" > etc/passwd
+      echo "nginx:x:1000:nginx" > etc/group
+    '';
+    config = {
+      Cmd = ["nginx" "-c" nginxConf];
+      ExposedPorts = {
+        "${nginxPort}/tcp" = {};
+      };
+    };
+  };
+}

tests/k8s/crd.nix

@@ -17,7 +17,7 @@ in {
       message = "should have group set";
       assertion = cfg.spec.group == "stable.example.com";
     }];
-    check = ''
+    testScript = ''
       $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
       $kube->succeed("kubectl get crds | grep -i crontabs");
       $kube->succeed("kubectl get crontabs | grep -i crontab");

tests/k8s/deployment.nix

@@ -1,56 +1,17 @@
-{ config, lib, pkgs, test, kubenix, ... }:
+{ config, lib, pkgs, test, kubenix, images, ... }:
 
 with lib;
 
 let
   cfg = config.kubernetes.api.deployments.nginx;
-
+  image = images.nginx;
-  nginxImage = let
-    nginxPort = "80";
-    nginxConf = pkgs.writeText "nginx.conf" ''
-      user nginx nginx;
-      daemon off;
-      error_log /dev/stdout info;
-      pid /dev/null;
-      events {}
-      http {
-        access_log /dev/stdout;
-        server {
-          listen ${nginxPort};
-          index index.html;
-          location / {
-            root ${nginxWebRoot};
-          }
-        }
-      }
-    '';
-    nginxWebRoot = pkgs.writeTextDir "index.html" ''
-      <html><body><h1>Hello from NGINX</h1></body></html>
-    '';
-  in pkgs.dockerTools.buildLayeredImage {
-    name = "xtruder/nginx";
-    tag = "latest";
-    contents = [pkgs.nginx];
-    extraCommands = ''
-      mkdir etc
-      chmod u+w etc
-      echo "nginx:x:1000:1000::/:" > etc/passwd
-      echo "nginx:x:1000:nginx" > etc/group
-    '';
-    config = {
-      Cmd = ["nginx" "-c" nginxConf];
-      ExposedPorts = {
-        "${nginxPort}/tcp" = {};
-      };
-    };
-  };
 in {
   imports = [
     kubenix.k8s
   ];
 
   test = {
-    name = "k8s-deployment-simple";
+    name = "k8s-deployment";
     description = "Simple k8s testing a simple deployment";
     assertions = [{
       message = "should have correct apiVersion and kind set";
@@ -59,8 +20,8 @@ in {
       message = "should have replicas set";
       assertion = cfg.spec.replicas == 10;
     }];
-    check = ''
+    testScript = ''
-      $kube->waitUntilSucceeds("docker load < ${nginxImage}");
+      $kube->waitUntilSucceeds("docker load < ${image}");
       $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
 
       $kube->succeed("kubectl get deployment | grep -i nginx");
@@ -76,7 +37,7 @@ in {
       template.metadata.labels.app = "nginx";
       template.spec = {
         containers.nginx = {
-          image = "xtruder/nginx:latest";
+          image = "${image.imageName}:${image.imageTag}";
           imagePullPolicy = "Never";
         };
       };

tests/submodules/simple.nix

@@ -10,7 +10,7 @@ in {
   ];
 
   test = {
-    name = "submodules/simple";
+    name = "submodules-simple";
     description = "Simple k8s submodule test";
     assertions = [{
       message = "Submodule name is set";