feat(testing): e2e testing improvements

testing/default.nix

@@ -33,12 +33,13 @@ let
         imports = [<nixpkgs/nixos/modules/profiles/minimal.nix>];
 
         config = mkMerge [{
-          virtualisation.cores = "all";
           boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
-          virtualisation.memorySize = mkDefault 1536;
+          virtualisation.memorySize = mkDefault 2048;
+          virtualisation.cores = mkDefault "all";
           virtualisation.diskSize = mkDefault 4096;
           networking = {
             inherit domain extraHosts;
+            nameservers = ["10.0.0.254"];
             primaryIPAddress = mkForce machine.ip;
 
             firewall = {
@@ -90,6 +91,109 @@ let
   } // attrs // {
     name = "kubernetes-${attrs.name}-singlenode";
   });
+
+  testOptions = {config, ...}: let
+    modules = [config.module ./test.nix {
+      config._module.args.test = config;
+    }] ++ cfg.defaults;
+
+    test = (kubenix.evalKubernetesModules {
+      check = false;
+      inherit modules;
+    }).config.test;
+
+    evaled =
+      if test.enable
+      then builtins.trace "testing ${test.name}" (kubenix.evalKubernetesModules {
+        inherit modules;
+      })
+      else {success = false;};
+  in {
+    options = {
+      name = mkOption {
+        description = "test name";
+        type = types.str;
+        internal = true;
+      };
+
+      description = mkOption {
+        description = "test description";
+        type = types.str;
+        internal = true;
+      };
+
+      enable = mkOption {
+        description = "Whether to enable test";
+        type = types.bool;
+        internal = true;
+      };
+
+      module = mkOption {
+        description = "Module defining submodule";
+        type = types.unspecified;
+      };
+
+      evaled = mkOption {
+        description = "Wheter test was evaled";
+        type = types.bool;
+        default =
+          if cfg.throwError
+          then if evaled.config.test.assertions != [] then true else true
+          else (builtins.tryEval evaled.config.test.assertions).success;
+        internal = true;
+      };
+
+      success = mkOption {
+        description = "Whether test was success";
+        type = types.bool;
+        internal = true;
+        default = false;
+      };
+
+      assertions = mkOption {
+        description = "Test result";
+        type = types.unspecified;
+        internal = true;
+        default = [];
+      };
+
+      script = mkOption {
+        description = "Test script";
+        type = types.nullOr types.package;
+        default = null;
+      };
+
+      driver = mkOption {
+        description = "Testing driver";
+        type = types.nullOr types.package;
+      };
+
+      generated = mkOption {
+        description = "Generated resources";
+        type = types.nullOr types.package;
+        default = null;
+      };
+    };
+
+    config = mkMerge [{
+      inherit (test) name description enable;
+    } (mkIf config.evaled {
+      inherit (evaled.config.test) assertions;
+      success = all (el: el.assertion) config.assertions;
+      script =
+        if cfg.e2e && evaled.config.test.check != null
+        then mkKubernetesSingleNodeTest {
+          name = config.name;
+          test = ''
+            $kube->waitUntilSucceeds("kubectl get node kube.my.zyx | grep -w Ready");
+            ${evaled.config.test.check}
+          '';
+        } else null;
+      driver = mkIf (config.script != null) config.script.driver;
+      generated = mkIf (hasAttr "kubernetes" evaled.config)
+        pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated);
+    })];
+  };
 in {
   options = {
     testing.throwError = mkOption {
@@ -98,6 +202,12 @@ in {
       default = true;
     };
 
+    testing.e2e = mkOption {
+      description = "Whether to enable e2e tests";
+      type = types.bool;
+      default = true;
+    };
+
     testing.defaults = mkOption {
       description = "Testing defaults";
       type = types.coercedTo types.unspecified (value: [value]) (types.listOf types.unspecified);
@@ -110,95 +220,16 @@ in {
     testing.tests = mkOption {
       description = "Attribute set of test cases";
       default = [];
-      type = types.listOf (types.coercedTo types.path (module: {inherit module;}) (types.submodule ({config, ...}: let
+      type = types.listOf (types.coercedTo types.path (module: {inherit module;}) (types.submodule testOptions));
-        modules = [config.module ./test.nix {
-          config._module.args.test = config;
-        }] ++ cfg.defaults;
-
-        test = (kubenix.evalKubernetesModules {
-          check = false;
-          inherit modules;
-        }).config.test;
-
-        evaled =
-          if test.enable
-          then builtins.trace "testing ${test.name}" (kubenix.evalKubernetesModules {
-            inherit modules;
-          })
-          else {success = false;};
-      in {
-        options = {
-          name = mkOption {
-            description = "test name";
-            type = types.str;
-            internal = true;
-          };
-
-          description = mkOption {
-            description = "test description";
-            type = types.str;
-            internal = true;
-          };
-
-          enable = mkOption {
-            description = "Whether to enable test";
-            type = types.bool;
-            internal = true;
-          };
-
-          module = mkOption {
-            description = "Module defining submodule";
-            type = types.unspecified;
-          };
-
-          evaled = mkOption {
-            description = "Wheter test was evaled";
-            type = types.bool;
-            default =
-              if cfg.throwError
-              then if evaled.config.test.assertions != [] then true else true
-              else (builtins.tryEval evaled.config.test.assertions).success;
-            internal = true;
-          };
-
-          success = mkOption {
-            description = "Whether test was success";
-            type = types.bool;
-            internal = true;
-            default = false;
-          };
-
-          assertions = mkOption {
-            description = "Test result";
-            type = types.unspecified;
-            internal = true;
-            default = [];
-          };
-
-          script = mkOption {
-            description = "Test script";
-            type = types.nullOr types.package;
-            default = null;
-          };
-        };
-
-        config = mkMerge [{
-          inherit (test) name description enable;
-        } (mkIf config.evaled {
-          inherit (evaled.config.test) assertions;
-          success = all (el: el.assertion) config.assertions;
-          script = if evaled.config.test.check != null then mkKubernetesSingleNodeTest {
-            name = config.name;
-            test = ''
-              $kube->waitUntilSucceeds("kubectl get node kube.my.zyx | grep -w Ready");
-              ${evaled.config.test.check}
-            '';
-          } else null;
-        })];
-      })));
       apply = tests: filter (test: test.enable) tests;
     };
 
+    testing.testsByName = mkOption {
+      description = "Tests by name";
+      type = types.attrsOf types.attrs;
+      default = listToAttrs (map (test: nameValuePair test.name test) cfg.tests);
+    };
+
     testing.success = mkOption {
       description = "Whether testing was a success";
       type = types.bool;

tests/default.nix

@@ -4,39 +4,46 @@
 , k8sVersions ? ["1.7" "1.8" "1.9" "1.10" "1.11" "1.12" "1.13"]
 
 # whether any testing error should throw an error
-, throwError ? true }:
+, throwError ? true
+, e2e ? true }:
 
 with lib;
 
-listToAttrs (map (version: let
+let
-  version' = replaceStrings ["."] ["_"] version;
+  tests = listToAttrs (map (version: let
-in nameValuePair "v${version'}" (evalModules {
+    version' = replaceStrings ["."] ["_"] version;
-  modules = [
+  in nameValuePair "v${version'}" (evalModules {
-    kubenix.testing
+    modules = [
+      kubenix.testing
 
-    {
+      {
-      imports = [kubenix.k8s kubenix.submodules];
+        imports = [kubenix.k8s kubenix.submodules];
 
-      kubernetes.version = version;
-
-      testing.throwError = throwError;
-      testing.tests = [
-        ./k8s/simple.nix
-        ./k8s/deployment.nix
-        ./k8s/crd.nix
-        ./k8s/1.13/crd.nix
-        ./submodules/simple.nix
-      ];
-      testing.defaults = ({kubenix, ...}: {
-        imports = [kubenix.k8s];
         kubernetes.version = version;
-      });
+
-    }
+        testing.throwError = throwError;
-  ];
+        testing.e2e = e2e;
-  args = {
+        testing.tests = [
-    inherit pkgs;
+          ./k8s/simple.nix
-  };
+          ./k8s/deployment.nix
-  specialArgs = {
+          ./k8s/crd.nix
-    inherit kubenix;
+          ./k8s/1.13/crd.nix
-  };
+          ./submodules/simple.nix
-}).config.testing.result) k8sVersions)
+        ];
+        testing.defaults = ({kubenix, ...}: {
+          imports = [kubenix.k8s];
+          kubernetes.version = version;
+        });
+      }
+    ];
+    args = {
+      inherit pkgs;
+    };
+    specialArgs = {
+      inherit kubenix;
+    };
+  }).config) k8sVersions);
+in {
+  inherit tests;
+  results = mapAttrs (_: test: test.testing.result) tests;
+}

tests/k8s/1.13/crd.nix

@@ -10,7 +10,7 @@ in {
   ];
 
   test = {
-    name = "k8s/1.13/crd";
+    name = "k8s-1.13-crd";
     description = "Simple test tesing CRD for k8s 1.13";
     enable = builtins.compareVersions config.kubernetes.version "1.13" >= 0;
     assertions = [{

tests/k8s/crd.nix

@@ -10,7 +10,7 @@ in {
   ];
 
   test = {
-    name = "k8s.crd";
+    name = "k8s-crd";
     description = "Simple test tesing CRD";
     enable = builtins.compareVersions config.kubernetes.version "1.8" >= 0;
     assertions = [{

tests/k8s/deployment.nix

@@ -1,14 +1,56 @@
-{ config, test, kubenix, ... }:
+{ config, lib, pkgs, test, kubenix, ... }:
+
+with lib;
 
 let
   cfg = config.kubernetes.api.deployments.nginx;
+
+  nginxImage = let
+    nginxPort = "80";
+    nginxConf = pkgs.writeText "nginx.conf" ''
+      user nginx nginx;
+      daemon off;
+      error_log /dev/stdout info;
+      pid /dev/null;
+      events {}
+      http {
+        access_log /dev/stdout;
+        server {
+          listen ${nginxPort};
+          index index.html;
+          location / {
+            root ${nginxWebRoot};
+          }
+        }
+      }
+    '';
+    nginxWebRoot = pkgs.writeTextDir "index.html" ''
+      <html><body><h1>Hello from NGINX</h1></body></html>
+    '';
+  in pkgs.dockerTools.buildLayeredImage {
+    name = "xtruder/nginx";
+    tag = "latest";
+    contents = [pkgs.nginx];
+    extraCommands = ''
+      mkdir etc
+      chmod u+w etc
+      echo "nginx:x:1000:1000::/:" > etc/passwd
+      echo "nginx:x:1000:nginx" > etc/group
+    '';
+    config = {
+      Cmd = ["nginx" "-c" nginxConf];
+      ExposedPorts = {
+        "${nginxPort}/tcp" = {};
+      };
+    };
+  };
 in {
   imports = [
     kubenix.k8s
   ];
 
   test = {
-    name = "k8s/deployment/simple";
+    name = "k8s-deployment-simple";
     description = "Simple k8s testing a simple deployment";
     assertions = [{
       message = "should have correct apiVersion and kind set";
@@ -17,6 +59,14 @@ in {
       message = "should have replicas set";
       assertion = cfg.spec.replicas == 10;
     }];
+    check = ''
+      $kube->waitUntilSucceeds("docker load < ${nginxImage}");
+      $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}");
+
+      $kube->succeed("kubectl get deployment | grep -i nginx");
+      $kube->waitUntilSucceeds("kubectl get deployment -o go-template nginx --template={{.status.readyReplicas}} | grep 10");
+      $kube->waitUntilSucceeds("${pkgs.curl}/bin/curl http://nginx.default.svc.cluster.local | grep -i hello");
+    '';
   };
 
   kubernetes.api.deployments.nginx = {
@@ -26,9 +76,20 @@ in {
       template.metadata.labels.app = "nginx";
       template.spec = {
         containers.nginx = {
-          image = "nginx";
+          image = "xtruder/nginx:latest";
+          imagePullPolicy = "Never";
         };
       };
     };
   };
+
+  kubernetes.api.services.nginx = {
+    spec = {
+      ports = [{
+        name = "http";
+        port = 80;
+      }];
+      selector.app = "nginx";
+    };
+  };
 }

tests/k8s/simple.nix

@@ -10,7 +10,7 @@ in {
   ];
 
   test = {
-    name = "k8s/simple";
+    name = "k8s-simple";
     description = "Simple k8s testing wheter name, apiVersion and kind are preset";
     assertions = [{
       message = "should have apiVersion and kind set";