feat: initial support for e2e tests

testing/default.nix

@@ -5,6 +5,91 @@ with lib;
 let
   cfg = config.testing;
   parentConfig = config;
+
+  nixosTesting = import <nixpkgs/nixos/lib/testing.nix> {
+    inherit pkgs;
+    system = "x86_64-linux";
+  };
+
+  mkKubernetesBaseTest =
+    { name, domain ? "my.zyx", test, machines
+    , extraConfiguration ? null }:
+    let
+      masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
+      master = machines.${masterName};
+      extraHosts = ''
+        ${master.ip}  etcd.${domain}
+        ${master.ip}  api.${domain}
+        ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
+      '';
+      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
+        mkdir -p $out/bin
+        makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
+      '';
+    in nixosTesting.makeTest {
+      inherit name;
+
+      nodes = mapAttrs (machineName: machine:
+        { config, pkgs, lib, nodes, ... }:
+          mkMerge [
+            {
+              boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+              virtualisation.memorySize = mkDefault 1536;
+              virtualisation.diskSize = mkDefault 4096;
+              networking = {
+                inherit domain extraHosts;
+                primaryIPAddress = mkForce machine.ip;
+
+                firewall = {
+                  allowedTCPPorts = [
+                    10250 # kubelet
+                  ];
+                  trustedInterfaces = ["docker0"];
+
+                  extraCommands = concatMapStrings  (node: ''
+                    iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+                  '') (attrValues nodes);
+                };
+              };
+              environment.systemPackages = [ kubectl ];
+              services.flannel.iface = "eth1";
+              services.kubernetes = {
+                easyCerts = true;
+                inherit (machine) roles;
+                apiserver = {
+                  securePort = 443;
+                  advertiseAddress = master.ip;
+                };
+                masterAddress = "${masterName}.${config.networking.domain}";
+              };
+            }
+            (optionalAttrs (any (role: role == "master") machine.roles) {
+              networking.firewall.allowedTCPPorts = [
+                443 # kubernetes apiserver
+              ];
+            })
+            (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
+            (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
+          ]
+      ) machines;
+
+      testScript = ''
+        startAll;
+
+        ${test}
+      '';
+    };
+
+  mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
+    machines = {
+      kube = {
+        roles = ["master" "node"];
+        ip = "192.168.1.1";
+      };
+    };
+  } // attrs // {
+    name = "kubernetes-${attrs.name}-singlenode";
+  });
 in {
   options = {
     testing.throwError = mkOption {
@@ -89,13 +174,27 @@ in {
             internal = true;
             default = [];
           };
+
+          script = mkOption {
+            description = "Test script";
+            type = types.nullOr types.package;
+            default = null;
+          };
         };
 
-        config = {
+        config = mkMerge [{
           inherit (test) name description enable;
-          assertions = mkIf config.evaled evaled.config.test.assertions;
-          success = mkIf config.evaled (all (el: el.assertion) config.assertions);
-        };
+        } (mkIf config.evaled {
+          inherit (evaled.config.test) assertions;
+          success = all (el: el.assertion) config.assertions;
+          script = if evaled.config.test.check != null then mkKubernetesSingleNodeTest {
+            name = config.name;
+            test = ''
+              $kube->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
+              ${evaled.config.test.check}
+            '';
+          } else null;
+        })];
       })));
       apply = tests: filter (test: test.enable) tests;
     };
@@ -114,6 +213,7 @@ in {
         tests = map (test: {
           inherit (test) name description evaled success;
           assertions = moduleToAttrs test.assertions;
+          script = test.script;
         }) (filter (test: test.enable) cfg.tests);
       });
     };

testing/test.nix

@@ -1,8 +1,10 @@
-{ lib, ... }:
+{ lib, config, pkgs, ... }:
 
 with lib;
 
-{
+let
+  cfg = config.test;
+in {
   options.test = {
     name = mkOption {
       description = "Test name";
@@ -43,5 +45,16 @@ with lib;
         succeed, along with associated error messages for the user.
       '';
     };
+
+    extraCheckInputs = mkOption {
+      description = "Extra check inputs";
+      type = types.listOf types.package;
+    };
+
+    check = mkOption {
+      description = "Script to run as part testing";
+      type = types.nullOr types.lines;
+      default = null;
+    };
   };
 }