feat: refactor testing

testing/default.nix

@@ -11,26 +11,20 @@ let
     system = "x86_64-linux";
   };
 
-  mkKubernetesBaseTest =
-    { name, domain ? "my.zyx", test, machines
-    , extraConfiguration ? null }:
-    let
-      masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
-      master = machines.${masterName};
+  kubernetesBaseConfig = { config, pkgs, lib, nodes, ... }: let
+    master = findFirst
+      (node: any (role: role == "master") node.config.services.kubernetes.roles)
+      (throw "no master node")
+      (attrValues nodes);
     extraHosts = ''
-        ${master.ip}  etcd.${domain}
-        ${master.ip}  api.${domain}
-        ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
+      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
+      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
+      ${concatMapStringsSep "\n"
+        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
+        (attrValues nodes)}
     '';
-      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
-        mkdir -p $out/bin
-        makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
-      '';
-    in nixosTesting.makeTest {
-      inherit name;
-
-      nodes = mapAttrs (machineName: machine: { config, pkgs, lib, nodes, ... }: {
-        imports = [<nixpkgs/nixos/modules/profiles/minimal.nix>];
+  in {
+    imports = [ <nixpkgs/nixos/modules/profiles/minimal.nix> ];
 
     config = mkMerge [{
       boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
@@ -38,10 +32,9 @@ let
       virtualisation.cores = mkDefault "all";
       virtualisation.diskSize = mkDefault 4096;
       networking = {
-            inherit domain extraHosts;
+        inherit extraHosts;
+        domain = "my.xzy";
         nameservers = ["10.0.0.254"];
-            primaryIPAddress = mkForce machine.ip;
-
         firewall = {
           allowedTCPPorts = [
             10250 # kubelet
@@ -53,44 +46,35 @@ let
           '') (attrValues nodes);
         };
       };
-          environment.systemPackages = [ kubectl ];
+      environment.systemPackages = [ pkgs.kubectl ];
+      environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
       services.flannel.iface = "eth1";
       services.kubernetes = {
         easyCerts = true;
-            inherit (machine) roles;
         apiserver = {
           securePort = 443;
-              advertiseAddress = master.ip;
+          advertiseAddress = master.config.networking.primaryIPAddress;
         };
-            masterAddress = "${masterName}.${config.networking.domain}";
+        masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
       };
     }
-        (optionalAttrs (any (role: role == "master") machine.roles) {
+    (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
       networking.firewall.allowedTCPPorts = [
         443 # kubernetes apiserver
       ];
-        })
-        (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
-        (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))];
-      }) machines;
-
-      testScript = ''
-        startAll;
-
-        ${test}
-      '';
+    })];
   };
 
-  mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
-    machines = {
-      kube = {
+  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
+    nixosTesting.makeTest {
+      inherit name;
+
+      nodes.kube = { config, pkgs, lib, nodes, ... }: {
+        imports = [ kubernetesBaseConfig ];
+        services.kubernetes = {
           roles = ["master" "node"];
-        ip = "192.168.1.1";
-      };
-    };
-    extraConfiguration = {...}: {
-      services.kubernetes.flannel.enable = false;
-      services.kubernetes.kubelet = {
+          flannel.enable = false;
+          kubelet = {
             networkPlugin = "cni";
             cni.config = [{
               name = "mynet";
@@ -110,9 +94,17 @@ let
             }];
           };
         };
-  } // attrs // {
-    name = "kubernetes-${attrs.name}-singlenode";
-  });
+        networking.primaryIPAddress = mkForce "192.168.1.1";
+      };
+
+      testScript = ''
+        startAll;
+
+        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
+
+        ${testScript}
+      '';
+    };
 
   testOptions = {config, ...}: let
     modules = [config.module ./test.nix {
@@ -179,17 +171,12 @@ let
         default = [];
       };
 
-      script = mkOption {
-        description = "Test script";
+      test = mkOption {
+        description = "Test derivation to run";
         type = types.nullOr types.package;
         default = null;
       };
 
-      driver = mkOption {
-        description = "Testing driver";
-        type = types.nullOr types.package;
-      };
-
       generated = mkOption {
         description = "Generated resources";
         type = types.nullOr types.package;
@@ -202,18 +189,14 @@ let
     } (mkIf config.evaled {
       inherit (evaled.config.test) assertions;
       success = all (el: el.assertion) config.assertions;
-      script =
-        if cfg.e2e && evaled.config.test.check != null
+      test =
+        if cfg.e2e && evaled.config.test.testScript != null
         then mkKubernetesSingleNodeTest {
+          inherit (evaled.config.test) testScript;
           name = config.name;
-          test = ''
-            $kube->waitUntilSucceeds("kubectl get node kube.my.zyx | grep -w Ready");
-            ${evaled.config.test.check}
-          '';
         } else null;
-      driver = mkIf (config.script != null) config.script.driver;
       generated = mkIf (hasAttr "kubernetes" evaled.config)
-        pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated);
+        (pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated));
     })];
   };
 in {
@@ -264,9 +247,8 @@ in {
       default = pkgs.writeText "testing-report.json" (builtins.toJSON {
         success = cfg.success;
         tests = map (test: {
-          inherit (test) name description evaled success;
+          inherit (test) name description evaled success test;
           assertions = moduleToAttrs test.assertions;
-          script = test.script;
         }) (filter (test: test.enable) cfg.tests);
       });
     };

testing/test.nix

@@ -51,10 +51,16 @@ in {
       type = types.listOf types.package;
     };
 
-    check = mkOption {
-      description = "Script to run as part testing";
+    testScript = mkOption {
+      description = "Script to run as part of testing";
       type = types.nullOr types.lines;
       default = null;
     };
+
+    extraConfig = mkOption {
+      description = "Extra configuration for running test";
+      type = types.unspecified;
+      default = {};
+    };
   };
 }

tests/default.nix

@@ -10,6 +10,8 @@
 with lib;
 
 let
+  images = pkgs.callPackage ./images.nix {};
+
   tests = listToAttrs (map (version: let
     version' = replaceStrings ["."] ["_"] version;
   in nameValuePair "v${version'}" (evalModules {
@@ -28,11 +30,13 @@ let
           ./k8s/deployment.nix
           ./k8s/crd.nix
           ./k8s/1.13/crd.nix
+          ./k8s/submodule.nix
           ./submodules/simple.nix
         ];
         testing.defaults = ({kubenix, ...}: {
           imports = [kubenix.k8s];
           kubernetes.version = version;
+          _module.args.images = images;
         });
       }
     ];
@@ -43,7 +47,4 @@ let
       inherit kubenix;
     };
   }).config) k8sVersions);
-in {
-  inherit tests;
-  results = mapAttrs (_: test: test.testing.result) tests;
-}
+in tests

tests/images.nix

@@ -0,0 +1,45 @@
+{ pkgs, dockerTools, lib, ... }:
+
+with lib;
+
+{
+  nginx = let
+    nginxPort = "80";
+    nginxConf = pkgs.writeText "nginx.conf" ''
+      user nginx nginx;
+      daemon off;
+      error_log /dev/stdout info;
+      pid /dev/null;
+      events {}
+      http {
+        access_log /dev/stdout;
+        server {
+          listen ${nginxPort};
+          index index.html;
+          location / {
+            root ${nginxWebRoot};
+          }
+        }
+      }
+    '';
+    nginxWebRoot = pkgs.writeTextDir "index.html" ''
+      <html><body><h1>Hello from NGINX</h1></body></html>
+    '';
+  in dockerTools.buildLayeredImage {
+    name = "xtruder/nginx";
+    tag = "latest";
+    contents = [pkgs.nginx];
+    extraCommands = ''
+      mkdir etc
+      chmod u+w etc
+      echo "nginx:x:1000:1000::/:" > etc/passwd
+      echo "nginx:x:1000:nginx" > etc/group
+    '';
+    config = {
+      Cmd = ["nginx" "-c" nginxConf];
+      ExposedPorts = {
+        "${nginxPort}/tcp" = {};
+      };
+    };
+  };
+}

tests/k8s/crd.nix

@@ -17,7 +17,7 @@ in {
       message = "should have group set";
       assertion = cfg.spec.group == "stable.example.com";
     }];
-    check = ''
+    testScript = ''
       $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
       $kube->succeed("kubectl get crds | grep -i crontabs");
       $kube->succeed("kubectl get crontabs | grep -i crontab");

tests/k8s/deployment.nix

@@ -1,56 +1,17 @@
-{ config, lib, pkgs, test, kubenix, ... }:
+{ config, lib, pkgs, test, kubenix, images, ... }:
 
 with lib;
 
 let
   cfg = config.kubernetes.api.deployments.nginx;
-
-  nginxImage = let
-    nginxPort = "80";
-    nginxConf = pkgs.writeText "nginx.conf" ''
-      user nginx nginx;
-      daemon off;
-      error_log /dev/stdout info;
-      pid /dev/null;
-      events {}
-      http {
-        access_log /dev/stdout;
-        server {
-          listen ${nginxPort};
-          index index.html;
-          location / {
-            root ${nginxWebRoot};
-          }
-        }
-      }
-    '';
-    nginxWebRoot = pkgs.writeTextDir "index.html" ''
-      <html><body><h1>Hello from NGINX</h1></body></html>
-    '';
-  in pkgs.dockerTools.buildLayeredImage {
-    name = "xtruder/nginx";
-    tag = "latest";
-    contents = [pkgs.nginx];
-    extraCommands = ''
-      mkdir etc
-      chmod u+w etc
-      echo "nginx:x:1000:1000::/:" > etc/passwd
-      echo "nginx:x:1000:nginx" > etc/group
-    '';
-    config = {
-      Cmd = ["nginx" "-c" nginxConf];
-      ExposedPorts = {
-        "${nginxPort}/tcp" = {};
-      };
-    };
-  };
+  image = images.nginx;
 in {
   imports = [
     kubenix.k8s
   ];
 
   test = {
-    name = "k8s-deployment-simple";
+    name = "k8s-deployment";
     description = "Simple k8s testing a simple deployment";
     assertions = [{
       message = "should have correct apiVersion and kind set";
@@ -59,8 +20,8 @@ in {
       message = "should have replicas set";
       assertion = cfg.spec.replicas == 10;
     }];
-    check = ''
-      $kube->waitUntilSucceeds("docker load < ${nginxImage}");
+    testScript = ''
+      $kube->waitUntilSucceeds("docker load < ${image}");
       $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
 
       $kube->succeed("kubectl get deployment | grep -i nginx");
@@ -76,7 +37,7 @@ in {
       template.metadata.labels.app = "nginx";
       template.spec = {
         containers.nginx = {
-          image = "xtruder/nginx:latest";
+          image = "${image.imageName}:${image.imageTag}";
           imagePullPolicy = "Never";
         };
       };

tests/submodules/simple.nix

@@ -10,7 +10,7 @@ in {
   ];
 
   test = {
-    name = "submodules/simple";
+    name = "submodules-simple";
     description = "Simple k8s submodule test";
     assertions = [{
       message = "Submodule name is set";