feat: initial support for e2e tests

lib.nix

@@ -27,11 +27,11 @@ in rec {
   loadYAML = path: importJSON (pkgs.runCommand "yaml-to-json" {
   } "${pkgs.remarshal}/bin/remarshal -i ${path} -if yaml -of json > $out");
 
-  toYAML = config: builtins.readFile (pkgs.runCommand "to-yaml" {
+  toYAML = config: pkgs.runCommand "to-yaml" {
     buildInputs = [pkgs.remarshal];
   } ''
     remarshal -i ${pkgs.writeText "to-json" (builtins.toJSON config)} -if json -of yaml > $out
-  '');
+  '';
 
   toBase64 = value:
     builtins.readFile

testing/default.nix

@@ -5,6 +5,91 @@ with lib;
 let
   cfg = config.testing;
   parentConfig = config;
+
+  nixosTesting = import <nixpkgs/nixos/lib/testing.nix> {
+    inherit pkgs;
+    system = "x86_64-linux";
+  };
+
+  mkKubernetesBaseTest =
+    { name, domain ? "my.zyx", test, machines
+    , extraConfiguration ? null }:
+    let
+      masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
+      master = machines.${masterName};
+      extraHosts = ''
+        ${master.ip}  etcd.${domain}
+        ${master.ip}  api.${domain}
+        ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
+      '';
+      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
+        mkdir -p $out/bin
+        makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
+      '';
+    in nixosTesting.makeTest {
+      inherit name;
+
+      nodes = mapAttrs (machineName: machine:
+        { config, pkgs, lib, nodes, ... }:
+          mkMerge [
+            {
+              boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+              virtualisation.memorySize = mkDefault 1536;
+              virtualisation.diskSize = mkDefault 4096;
+              networking = {
+                inherit domain extraHosts;
+                primaryIPAddress = mkForce machine.ip;
+
+                firewall = {
+                  allowedTCPPorts = [
+                    10250 # kubelet
+                  ];
+                  trustedInterfaces = ["docker0"];
+
+                  extraCommands = concatMapStrings  (node: ''
+                    iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+                  '') (attrValues nodes);
+                };
+              };
+              environment.systemPackages = [ kubectl ];
+              services.flannel.iface = "eth1";
+              services.kubernetes = {
+                easyCerts = true;
+                inherit (machine) roles;
+                apiserver = {
+                  securePort = 443;
+                  advertiseAddress = master.ip;
+                };
+                masterAddress = "${masterName}.${config.networking.domain}";
+              };
+            }
+            (optionalAttrs (any (role: role == "master") machine.roles) {
+              networking.firewall.allowedTCPPorts = [
+                443 # kubernetes apiserver
+              ];
+            })
+            (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
+            (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
+          ]
+      ) machines;
+
+      testScript = ''
+        startAll;
+
+        ${test}
+      '';
+    };
+
+  mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
+    machines = {
+      kube = {
+        roles = ["master" "node"];
+        ip = "192.168.1.1";
+      };
+    };
+  } // attrs // {
+    name = "kubernetes-${attrs.name}-singlenode";
+  });
 in {
   options = {
     testing.throwError = mkOption {
@@ -89,13 +174,27 @@ in {
             internal = true;
             default = [];
           };
+
+          script = mkOption {
+            description = "Test script";
+            type = types.nullOr types.package;
+            default = null;
+          };
         };
 
-        config = {
+        config = mkMerge [{
           inherit (test) name description enable;
-          assertions = mkIf config.evaled evaled.config.test.assertions;
+        } (mkIf config.evaled {
-          success = mkIf config.evaled (all (el: el.assertion) config.assertions);
+          inherit (evaled.config.test) assertions;
-        };
+          success = all (el: el.assertion) config.assertions;
+          script = if evaled.config.test.check != null then mkKubernetesSingleNodeTest {
+            name = config.name;
+            test = ''
+              $kube->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
+              ${evaled.config.test.check}
+            '';
+          } else null;
+        })];
       })));
       apply = tests: filter (test: test.enable) tests;
     };
@@ -114,6 +213,7 @@ in {
         tests = map (test: {
           inherit (test) name description evaled success;
           assertions = moduleToAttrs test.assertions;
+          script = test.script;
         }) (filter (test: test.enable) cfg.tests);
       });
     };

testing/test.nix

@@ -1,8 +1,10 @@
-{ lib, ... }:
+{ lib, config, pkgs, ... }:
 
 with lib;
 
-{
+let
+  cfg = config.test;
+in {
   options.test = {
     name = mkOption {
       description = "Test name";
@@ -43,5 +45,16 @@ with lib;
         succeed, along with associated error messages for the user.
       '';
     };
+
+    extraCheckInputs = mkOption {
+      description = "Extra check inputs";
+      type = types.listOf types.package;
+    };
+
+    check = mkOption {
+      description = "Script to run as part testing";
+      type = types.nullOr types.lines;
+      default = null;
+    };
   };
 }

tests/k8s/crd.nix

@@ -1,4 +1,4 @@
-{ config, lib, kubenix, ... }:
+{ config, lib, kubenix, pkgs, ... }:
 
 with lib;
 
@@ -10,13 +10,18 @@ in {
   ];
 
   test = {
-    name = "k8s/crd";
+    name = "k8s.crd";
     description = "Simple test tesing CRD";
     enable = builtins.compareVersions config.kubernetes.version "1.8" >= 0;
     assertions = [{
       message = "should have group set";
       assertion = cfg.spec.group == "stable.example.com";
     }];
+    check = ''
+      $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
+      $kube->succeed("kubectl get crds | grep -i crontabs");
+      $kube->succeed("kubectl get crontabs | grep -i crontab");
+    '';
   };
 
   kubernetes.api.customresourcedefinitions.crontabs = {
@@ -33,4 +38,20 @@ in {
       };
     };
   };
+
+  kubernetes.customResources = [{
+    group = "stable.example.com";
+    version = "v1";
+    kind = "CronTab";
+    plural = "crontabs";
+    description = "CronTabs resources";
+    module = {
+      options.schedule = mkOption {
+        description = "Crontab schedule script";
+        type = types.str;
+      };
+    };
+  }];
+
+  kubernetes.api."stable.example.com"."v1".CronTab.crontab.spec.schedule = "* * * * *";
 }

tests/k8s/simple.nix

@@ -1,4 +1,6 @@
-{ config, test, kubenix, ... }:
+{ config, test, kubenix, k8s, ... }:
+
+with k8s;
 
 let
   cfg = config.kubernetes.api.pods.nginx;