From bbc5e3d477d453a05d5c7de1f9ec8446aa7742cb Mon Sep 17 00:00:00 2001
From: Jaka Hudoklin <jaka@x-truder.net>
Date: Sun, 5 Apr 2020 21:25:34 +0700
Subject: [PATCH] WIP: test refactoring

---
 lib/docker.nix                                |   6 +-
 lib/extra.nix                                 |   6 +
 modules/default.nix                           |   4 +-
 modules/k8s.nix                               |  22 ++-
 modules/testing/default.nix                   |  99 +++++++++++++
 modules/testing/docker.nix                    |  46 ++++++
 modules/testing/driver/kubetest.nix           |  51 +++++++
 .../testing/driver/kubetest/test-options.nix  |  28 ++++
 .../driver/kubetest/testing-options.nix       |  15 ++
 modules/testing/runtime/local.nix             |  65 +++++++++
 modules/testing/runtime/nixos-k8s.nix         | 130 +++++++++++++++++
 .../runtime/nixos-k8s/runtime-options.nix     |   7 +
 .../{test.nix => testing/test-options.nix}    |  17 ++-
 modules/testing/test.nix                      | 134 ++++++++++++++++++
 tests/default.nix                             |  74 ++++++----
 tests/images.nix                              |   7 +
 tests/k8s/deployment.nix                      |  69 ++++++---
 17 files changed, 714 insertions(+), 66 deletions(-)
 create mode 100644 modules/testing/default.nix
 create mode 100644 modules/testing/docker.nix
 create mode 100644 modules/testing/driver/kubetest.nix
 create mode 100644 modules/testing/driver/kubetest/test-options.nix
 create mode 100644 modules/testing/driver/kubetest/testing-options.nix
 create mode 100644 modules/testing/runtime/local.nix
 create mode 100644 modules/testing/runtime/nixos-k8s.nix
 create mode 100644 modules/testing/runtime/nixos-k8s/runtime-options.nix
 rename modules/{test.nix => testing/test-options.nix} (82%)
 create mode 100644 modules/testing/test.nix

diff --git a/lib/docker.nix b/lib/docker.nix
index 8e7451f..c7ff120 100644
--- a/lib/docker.nix
+++ b/lib/docker.nix
@@ -4,12 +4,12 @@ with lib;
 
 {
   copyDockerImages = { images,  dest, args ? "" }:
-    pkgs.writeScriptBin "copy-docker-images" (concatMapStrings (image: ''
-      #!${pkgs.bash}/bin/bash
+    pkgs.writeScript "copy-docker-images.sh" (concatMapStrings (image: ''
+      #!${pkgs.runtimeShell}
 
       set -e
 
-      echo "copying ${image.imageName}:${image.imageTag}"
+      echo "copying '${image.imageName}:${image.imageTag}' to '${dest}/${image.imageName}:${image.imageTag}'"
       ${pkgs.skopeo}/bin/skopeo copy ${args} $@ docker-archive:${image} ${dest}/${image.imageName}:${image.imageTag}
     '') images);
 }
diff --git a/lib/extra.nix b/lib/extra.nix
index 2a69eea..72cac9d 100644
--- a/lib/extra.nix
+++ b/lib/extra.nix
@@ -30,6 +30,12 @@ rec {
     remarshal -i ${pkgs.writeText "to-json" (builtins.toJSON config)} -if json -of yaml > $out
   '');
 
+  toMultiDocumentYaml = name: documents: pkgs.runCommand name {
+    buildInputs = [ pkgs.remarshal ];
+  } (concatMapStringsSep "\necho --- >> $out\n" (d:
+    "remarshal -i ${builtins.toFile "doc" (builtins.toJSON d)} -if json -of yaml >> $out"
+  ) documents);
+
   toBase64 = value:
     builtins.readFile
       (pkgs.runCommand "value-to-b64" {} "echo -n '${value}' | ${pkgs.coreutils}/bin/base64 -w0 > $out");
diff --git a/modules/default.nix b/modules/default.nix
index ad574b1..faac82f 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -5,8 +5,8 @@
   submodule = ./submodule.nix;
   helm = ./helm.nix;
   docker = ./docker.nix;
-  testing = ./testing.nix;
-  test = ./test.nix;
+  testing = ./testing;
+  test = ./testing/test-options.nix;
   module = ./module.nix;
   legacy = ./legacy.nix;
 }
diff --git a/modules/k8s.nix b/modules/k8s.nix
index 9413d48..1dd6ef4 100644
--- a/modules/k8s.nix
+++ b/modules/k8s.nix
@@ -207,8 +207,8 @@ in {
 
     namespace = mkOption {
       description = "Default namespace where to deploy kubernetes resources";
-      type = types.str;
-      default = "default";
+      type = types.nullOr types.str;
+      default = null;
     };
 
     resourceOrder = mkOption {
@@ -308,6 +308,11 @@ in {
       description = "Generated kubernetes JSON file";
       type = types.package;
     };
+
+    resultYAML = mkOption {
+      description = "Genrated kubernetes YAML file";
+      type = types.package;
+    };
   };
 
   config = {
@@ -355,10 +360,14 @@ in {
       defaults = [{
         default = {
           # set default kubernetes namespace to all resources
-          metadata.namespace = mkDefault config.kubernetes.namespace;
+          metadata.namespace = mkIf (config.kubernetes.namespace != null)
+            (mkDefault config.kubernetes.namespace);
 
           # set project name to all resources
-          metadata.labels."kubenix/project-name" = config.kubenix.project;
+          metadata.annotations = {
+            "kubenix/project-name" = config.kubenix.project;
+            "kubenix/k8s-version" = cfg.version;
+          };
         };
       }];
     }] ++
@@ -390,6 +399,9 @@ in {
     };
 
     kubernetes.result =
-      pkgs.writeText "kubenix-generated.json" (builtins.toJSON cfg.generated);
+      pkgs.writeText "${config.kubenix.project}-generated.json" (builtins.toJSON cfg.generated);
+
+    kubernetes.resultYAML =
+      toMultiDocumentYaml "${config.kubenix.project}-generated.yaml" (config.kubernetes.objects);
   };
 }
diff --git a/modules/testing/default.nix b/modules/testing/default.nix
new file mode 100644
index 0000000..5651437
--- /dev/null
+++ b/modules/testing/default.nix
@@ -0,0 +1,99 @@
+{ nixosPath, config, pkgs, lib, kubenix, ... }:
+
+with lib;
+
+let
+  cfg = config.testing;
+
+  testModule = {
+    imports = [ ./test.nix ];
+
+    # passthru testing configuration
+    config._module.args = {
+      inherit pkgs kubenix;
+      testing = cfg;
+    };
+  };
+
+  isTestEnabled = test:
+    (cfg.enabledTests == null || elem test.name cfg.enabledTests) && test.enable;
+
+in {
+  imports = [
+    ./docker.nix
+    ./driver/kubetest.nix
+    ./runtime/local.nix
+  ];
+
+  options.testing = {
+    name = mkOption {
+      description = "Testing suite name";
+      type = types.str;
+      default = "default";
+    };
+
+    throwError = mkOption {
+      description = "Whether to throw error";
+      type = types.bool;
+      default = true;
+    };
+
+    defaults = mkOption {
+      description = "List of defaults to apply to tests";
+      type = types.listOf (types.submodule ({config, ...}: {
+        options = {
+          features = mkOption {
+            description = "List of features that test has to have to apply defaults";
+            type = types.listOf types.str;
+            default = [];
+          };
+
+          default = mkOption {
+            description = "Default to apply to test";
+            type = types.unspecified;
+            default = {};
+          };
+        };
+      }));
+      default = [];
+    };
+
+    tests = mkOption {
+      description = "List of test cases";
+      default = [];
+      type = types.listOf (types.coercedTo types.path (module: {
+        inherit module;
+      }) (types.submodule testModule));
+      apply = tests: filter isTestEnabled tests;
+    };
+
+    testsByName = mkOption {
+      description = "Tests by name";
+      type = types.attrsOf types.attrs;
+      default = listToAttrs (map (test: nameValuePair test.name test) cfg.tests);
+    };
+
+    enabledTests = mkOption {
+      description = "List of enabled tests (by default all tests are enabled)";
+      type = types.nullOr (types.listOf types.str);
+      default = null;
+    };
+
+    args = mkOption {
+      description = "Attribute set of extra args passed to tests";
+      type = types.attrs;
+      default = {};
+    };
+
+    success = mkOption {
+      description = "Whether testing was a success";
+      type = types.bool;
+      default = all (test: test.success) cfg.tests;
+    };
+
+    testScript = mkOption {
+      type = types.package;
+      description = "Script to run e2e tests";
+    };
+  };
+}
diff --git a/modules/testing/docker.nix b/modules/testing/docker.nix
new file mode 100644
index 0000000..afa66b5
--- /dev/null
+++ b/modules/testing/docker.nix
@@ -0,0 +1,46 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+with import ../../lib/docker.nix { inherit lib pkgs;  };
+
+let
+  testing = config.testing;
+
+  allImages = flatten (map (t: t.evaled.config.docker.export or []) testing.tests);
+
+  cfg = config.testing.docker;
+
+in {
+  options.testing.docker = {
+    registryUrl = mkOption {
+      description = "Docker registry url";
+      type = types.str;
+    };
+
+    images = mkOption {
+      description = "List of images to export";
+      type = types.listOf types.package;
+    };
+
+    copyScript = mkOption {
+      description = "Script to copy images to registry";
+      type = types.package;
+    };
+  };
+
+  config.testing.docker = {
+    images = allImages;
+
+    copyScript = copyDockerImages {
+      images = cfg.images;
+      dest = "docker://" + cfg.registryUrl;
+    };
+  };
+
+  config.testing.defaults = [{
+    features = ["docker"];
+    default = {
+      docker.registry.url = cfg.registryUrl;
+    };
+  }];
+}
diff --git a/modules/testing/driver/kubetest.nix b/modules/testing/driver/kubetest.nix
new file mode 100644
index 0000000..8214dde
--- /dev/null
+++ b/modules/testing/driver/kubetest.nix
@@ -0,0 +1,51 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  testing = config.testing;
+  cfg = testing.driver.kubetest;
+
+  pythonEnv = pkgs.python37.withPackages (ps: with ps; [
+    pytest
+    kubetest
+    kubernetes
+  ] ++ cfg.extraPackages);
+
+  toTestScript = t:
+    if isString t.script
+    then pkgs.writeText "${t.name}.py" ''
+      ${cfg.defaultHeader}
+      ${t.script}
+    ''
+    else p.script;
+
+  tests = pkgs.linkFarm "${testing.name}-tests" (map (t: {
+    path = toTestScript t;
+    name = "${t.name}_test.py";
+  }) testing.tests);
+
+  testScript = pkgs.writeScript "test-${testing.name}.sh" ''
+    #!/usr/bin/env bash
+    ${pythonEnv}/bin/pytest -p no:cacheprovider ${tests} $@
+  '';
+
+in {
+  options.testing.driver.kubetest = {
+    defaultHeader = mkOption {
+      type = types.lines;
+      description = "Default test header";
+      default = ''
+        import pytest
+      '';
+    };
+
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      description = "Extra packages to pass to tests";
+      default = [];
+    };
+  };
+
+  config.testing.testScript = testScript;
+}
diff --git a/modules/testing/driver/kubetest/test-options.nix b/modules/testing/driver/kubetest/test-options.nix
new file mode 100644
index 0000000..2206695
--- /dev/null
+++ b/modules/testing/driver/kubetest/test-options.nix
@@ -0,0 +1,28 @@
+{ lib, config, ... }:
+
+with lib;
+
+let
+  cfg = config.kubetest;
+
+in {
+  options.test.kubetest = {
+    enable = mkOption {
+      description = "Whether to use kubetest test driver";
+      type = types.bool;
+      default = cfg.testScript != "";
+    };
+
+    testScript = mkOption {
+      type = types.lines;
+      description = "Test script to use for kubetest";
+      default = "";
+    };
+
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      description = "List of extra packages to use for kubetest";
+      default = [];
+    };
+  };
+}
diff --git a/modules/testing/driver/kubetest/testing-options.nix b/modules/testing/driver/kubetest/testing-options.nix
new file mode 100644
index 0000000..6d5a985
--- /dev/null
+++ b/modules/testing/driver/kubetest/testing-options.nix
@@ -0,0 +1,15 @@
+{ lib, config, ... }:
+
+with lib;
+
+{
+  options.testing.kubetest = {
+    defaultHeader = mkOption {
+      description = "Default test header";
+      type = types.lines;
+      default = ''
+        import pytest
+      '';
+    };
+  };
+}
diff --git a/modules/testing/runtime/local.nix b/modules/testing/runtime/local.nix
new file mode 100644
index 0000000..bb17380
--- /dev/null
+++ b/modules/testing/runtime/local.nix
@@ -0,0 +1,65 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  testing = config.testing;
+
+  script = pkgs.writeScript "run-local-k8s-tests-${testing.name}.sh" ''
+    #!${pkgs.runtimeShell}
+
+    set -e
+
+    KUBECONFIG=''${KUBECONFIG:-~/.kube/config}
+    SKOPEOARGS=""
+
+    while (( "$#" )); do
+      case "$1" in
+        --kubeconfig)
+          KUBECONFIG=$2
+          shift 2
+          ;;
+        --skopeo-args)
+          SKOPEOARGS=$2
+          shift 2
+          ;;
+      esac
+    done
+
+    echo "--> copying docker images to registry"
+    ${testing.docker.copyScript} $SKOPEOARGS
+
+    echo "--> running tests"
+    ${testing.testScript} --kube-config=$KUBECONFIG
+  '';
+in {
+  options.testing.runtime.local = {
+    script = mkOption {
+      type = types.package;
+      description = "Runtime script";
+    };
+
+    docker = {
+      registryUrl = mkOption {
+        type = types.str;
+        description = "Docker registry url";
+      };
+
+      copyScript = mkOption {
+        type = types.package;
+        description = "Script used to copy docker images";
+      };
+    };
+    copyImages = mkOption {
+      type = types.package;
+      description = "Script used to copy docker images";
+    };
+
+    registryUrl = mkOption {
+      type = types.str;
+      description = "Registry url to copy docker images";
+    };
+  };
+
+  config.testing.runtime.local.script = script;
+}
diff --git a/modules/testing/runtime/nixos-k8s.nix b/modules/testing/runtime/nixos-k8s.nix
new file mode 100644
index 0000000..31b5121
--- /dev/null
+++ b/modules/testing/runtime/nixos-k8s.nix
@@ -0,0 +1,130 @@
+# nixos-k8s implements nixos kubernetes testing runtime
+
+{ nixosPath, config, pkgs, lib, kubenix, ... }:
+
+let
+  testConfig = config.evaled.config;
+
+  nixosTesting = import "${nixosPath}/lib/testing.nix" {
+    inherit pkgs;
+    system = "x86_64-linux";
+  };
+
+  kubernetesBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
+    master = findFirst
+      (node: any (role: role == "master") node.config.services.kubernetes.roles)
+      (throw "no master node")
+      (attrValues nodes);
+    extraHosts = ''
+      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
+      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
+      ${concatMapStringsSep "\n"
+        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
+        (attrValues nodes)}
+    '';
+  in {
+    imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
+
+    config = mkMerge [
+      # base configuration for master and nodes
+      {
+        boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+        virtualisation.memorySize = mkDefault 2048;
+        virtualisation.cores = mkDefault 16;
+        virtualisation.diskSize = mkDefault 4096;
+        networking = {
+          inherit extraHosts;
+          domain = "my.xzy";
+          nameservers = ["10.0.0.254"];
+          firewall = {
+            allowedTCPPorts = [
+              10250 # kubelet
+            ];
+            trustedInterfaces = ["docker0" "cni0"];
+
+            extraCommands = concatMapStrings  (node: ''
+              iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+            '') (attrValues nodes);
+          };
+        };
+        environment.systemPackages = [ pkgs.kubectl ];
+        environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
+        services.flannel.iface = "eth1";
+        services.kubernetes = {
+          easyCerts = true;
+          apiserver = {
+            securePort = 443;
+            advertiseAddress = master.config.networking.primaryIPAddress;
+          };
+          masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
+          seedDockerImages = mkIf (elem "docker" testConfig._m.features) testConfig.docker.export;
+        };
+
+        systemd.extraConfig = "DefaultLimitNOFILE=1048576";
+      }
+
+      # configuration only applied on master nodes
+      (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
+        networking.firewall.allowedTCPPorts = [
+          443 # kubernetes apiserver
+        ];
+      })
+    ];
+  };
+
+  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
+    nixosTesting.makeTest {
+      inherit name;
+
+      nodes.kube = { config, pkgs, nodes, ... }: {
+        imports = [ kubernetesBaseConfig extraConfiguration ];
+        services.kubernetes = {
+          roles = ["master" "node"];
+          flannel.enable = false;
+          kubelet = {
+            networkPlugin = "cni";
+            cni.config = [{
+              name = "mynet";
+              type = "bridge";
+              bridge = "cni0";
+              addIf = true;
+              ipMasq = true;
+              isGateway = true;
+              ipam = {
+                type = "host-local";
+                subnet = "10.1.0.0/16";
+                gateway = "10.1.0.1";
+                routes = [{
+                  dst = "0.0.0.0/0";
+                }];
+              };
+            }];
+          };
+        };
+        networking.primaryIPAddress = mkForce "192.168.1.1";
+      };
+
+      testScript = ''
+        startAll;
+
+        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
+
+        ${testScript}
+      '';
+    };
+
+in {
+  options = {
+    runtime.nixos-k8s = {
+      driver = mkOption {
+        description = "Test driver";
+        type = types.package;
+        internal = true;
+      };
+    };
+
+    runtime.nixos-k8s.driver = mkKubernetesSingleNodeTest {
+      inherit (config) name testScript;
+    };
+  };
+}
diff --git a/modules/testing/runtime/nixos-k8s/runtime-options.nix b/modules/testing/runtime/nixos-k8s/runtime-options.nix
new file mode 100644
index 0000000..894a1ed
--- /dev/null
+++ b/modules/testing/runtime/nixos-k8s/runtime-options.nix
@@ -0,0 +1,7 @@
+{ config, ... }:
+
+{
+  options.runtime.nixos-k8s = {
+
+  };
+}
diff --git a/modules/test.nix b/modules/testing/test-options.nix
similarity index 82%
rename from modules/test.nix
rename to modules/testing/test-options.nix
index 149d698..296a2c1 100644
--- a/modules/test.nix
+++ b/modules/testing/test-options.nix
@@ -4,6 +4,7 @@ with lib;
 
 let
   cfg = config.test;
+
 in {
   options.test = {
     name = mkOption {
@@ -38,7 +39,7 @@ in {
         };
       });
       default = [];
-      example = [ { assertion = false; message = "you can't enable this for that reason"; } ];
+      example = [ { assertion = false; message = "you can't enable this for some reason"; } ];
       description = ''
         This option allows modules to express conditions that must
         hold for the evaluation of the system configuration to
@@ -46,10 +47,9 @@ in {
       '';
     };
 
-    extraCheckInputs = mkOption {
-      description = "Extra check inputs";
-      type = types.listOf types.package;
-      default = [];
+    script = mkOption {
+      description = "Test script to use for e2e test";
+      type = types.nullOr (types.either types.lines types.path);
     };
 
     testScript = mkOption {
@@ -64,10 +64,9 @@ in {
       default = null;
     };
 
-    extraConfiguration = mkOption {
-      description = "Extra configuration for running test";
-      type = types.unspecified;
-      default = {};
+    driver = mkOption {
+      description = "Name of the driver to use for testing";
+      type = types.str;
     };
   };
 }
diff --git a/modules/testing/test.nix b/modules/testing/test.nix
new file mode 100644
index 0000000..38185e5
--- /dev/null
+++ b/modules/testing/test.nix
@@ -0,0 +1,134 @@
+{ lib, config, testing, kubenix, ... }:
+
+with lib;
+
+let
+  modules = [
+    # testing module
+    config.module
+
+    ./test-options.nix
+    ../base.nix
+
+    # passthru some options to test
+    {
+      config = {
+        kubenix.project = mkDefault config.name;
+        _module.args = {
+          inherit kubenix;
+          test = config;
+        } // testing.args;
+      };
+    }
+  ];
+
+  # eval without checking
+  evaled' = kubenix.evalModules {
+    check = false;
+    inherit modules;
+  };
+
+  # test configuration
+  testConfig = evaled'.config.test;
+
+  # test features
+  testFeatures = evaled'.config._m.features;
+
+  # defaults that can be applied on tests
+  defaults =
+    filter (d:
+      (intersectLists d.features testFeatures) == d.features ||
+      (length d.features) == 0
+    ) testing.defaults;
+
+  # add default modules to all modules
+  modulesWithDefaults = modules ++ (map (d: d.default) defaults);
+
+  # evaled test
+  evaled = let
+    evaled' = kubenix.evalModules {
+      modules = modulesWithDefaults;
+    };
+  in
+    if testing.throwError then evaled'
+    else if (builtins.tryEval evaled'.config.test.assertions).success
+    then evaled' else null;
+
+in {
+  imports = [
+    ./driver/kubetest.nix
+  ];
+
+  options = {
+    name = mkOption {
+      description = "test name";
+      type = types.str;
+      internal = true;
+    };
+
+    description = mkOption {
+      description = "test description";
+      type = types.str;
+      internal = true;
+    };
+
+    enable = mkOption {
+      description = "Whether to enable test";
+      type = types.bool;
+      internal = true;
+    };
+
+    module = mkOption {
+      description = "Module defining kubenix test";
+      type = types.unspecified;
+    };
+
+    evaled = mkOption {
+      description = "Test evaulation result";
+      type = types.nullOr types.attrs;
+      internal = true;
+    };
+
+    success = mkOption {
+      description = "Whether test assertions were successfull";
+      type = types.bool;
+      internal = true;
+      default = false;
+    };
+
+    assertions = mkOption {
+      description = "Test result";
+      type = types.unspecified;
+      internal = true;
+      default = [];
+    };
+
+    script = mkOption {
+      description = "Test script to use for e2e test";
+      type = types.nullOr (types.either types.lines types.path);
+      internal = true;
+    };
+
+    driver = mkOption {
+      description = "Name of the driver to use for testing";
+      type = types.str;
+      internal = true;
+    };
+  };
+
+  config = mkMerge [
+    {
+      inherit evaled;
+      inherit (testConfig) name description enable driver;
+    }
+
+    # if test is evaled check assertions
+    (mkIf (config.evaled != null) {
+      inherit (evaled.config.test) assertions;
+
+      # if all assertions are true, test is successfull
+      success = all (el: el.assertion) config.assertions;
+      script = evaled.config.test.script;
+    })
+  ];
+}
diff --git a/tests/default.nix b/tests/default.nix
index e43a5ed..8ab2982 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -1,48 +1,60 @@
 { pkgs ? import <nixpkgs> {}
 , lib ? pkgs.lib
 , kubenix ? import ../. { inherit pkgs lib; }
-, k8sVersion ? "1.21"
+
 , nixosPath ? toString <nixpkgs/nixos>
 
-# whether any testing error should throw an error
-, throwError ? true
-, e2e ? true }:
+, k8sVersion ? "1.18"
+, registryUrl ? throw "Registry url not defined"
+, throwError ? true # whether any testing error should throw an error
+, enabledTests ? null }:
 
 with lib;
 
 let
   images = pkgs.callPackage ./images.nix {};
 
-  test = (kubenix.evalModules {
+  config = (kubenix.evalModules {
     modules = [
       kubenix.modules.testing
 
       {
-        testing.name = "k8s-${k8sVersion}";
-        testing.throwError = throwError;
-        testing.e2e = e2e;
-        testing.tests = [
-          ./k8s/simple.nix
-          ./k8s/deployment.nix
-          ./k8s/deployment-k3s.nix
-          #  ./k8s/crd.nix # flaky
-          ./k8s/defaults.nix
-          ./k8s/order.nix
-          ./k8s/submodule.nix
-          ./k8s/imports.nix
-             # ./legacy/k8s.nix
-             # ./legacy/crd.nix
-             # ./legacy/modules.nix
-          ./helm/simple.nix
-             # ./istio/bookinfo.nix # infinite recusion
-          ./submodules/simple.nix
-          ./submodules/defaults.nix
-          ./submodules/versioning.nix
-          ./submodules/exports.nix
-          ./submodules/passthru.nix
-        ];
-        testing.args = {
-          inherit images k8sVersion;
+        testing = {
+          name = "kubenix-${k8sVersion}";
+          throwError = throwError;
+          enabledTests = enabledTests;
+          tests = [
+            ./k8s/simple.nix
+            ./k8s/deployment.nix
+            ./k8s/deployment-k3s.nix
+            #  ./k8s/crd.nix # flaky
+            ./k8s/defaults.nix
+            ./k8s/order.nix
+            ./k8s/submodule.nix
+            ./k8s/imports.nix
+            #./legacy/k8s.nix
+            #./legacy/crd.nix
+            #./legacy/modules.nix
+            ./helm/simple.nix
+            #  ./istio/bookinfo.nix # infinite recursion
+            ./submodules/simple.nix
+            ./submodules/defaults.nix
+            ./submodules/versioning.nix
+            ./submodules/exports.nix
+            ./submodules/passthru.nix
+          ];
+          args = {
+            inherit images;
+          };
+          docker.registryUrl = registryUrl;
+          defaults = [
+            {
+              features = ["k8s"];
+              default = {
+                kubernetes.version = k8sVersion;
+              };
+            }
+          ];
         };
       }
     ];
@@ -53,4 +65,4 @@ let
       inherit kubenix nixosPath;
     };
   }).config;
-in pkgs.recurseIntoAttrs test.testing
+in pkgs.recurseIntoAttrs config.testing
diff --git a/tests/images.nix b/tests/images.nix
index 4d2ac6a..8b16e90 100644
--- a/tests/images.nix
+++ b/tests/images.nix
@@ -3,6 +3,13 @@
 with lib;
 
 {
+  curl = dockerTools.buildLayeredImage {
+    name = "curl";
+    tag = "latest";
+    config.Cmd = [ "${pkgs.bash}" "-c" "sleep infinity" ];
+    contents = [ pkgs.bash pkgs.curl pkgs.cacert ];
+  };
+
   nginx = let
     nginxPort = "80";
     nginxConf = pkgs.writeText "nginx.conf" ''
diff --git a/tests/k8s/deployment.nix b/tests/k8s/deployment.nix
index 7e38a64..af8a8eb 100644
--- a/tests/k8s/deployment.nix
+++ b/tests/k8s/deployment.nix
@@ -1,10 +1,26 @@
-{ config, lib, pkgs, kubenix, images, k8sVersion, ... }:
+{ config, lib, pkgs, kubenix, images, ... }:
 
 with lib;
 
 let
   cfg = config.kubernetes.api.resources.deployments.nginx;
   image = images.nginx;
+
+  clientPod = builtins.toFile "client.json" (builtins.toJSON {
+    apiVersion = "v1";
+    kind = "Pod";
+    metadata = {
+      namespace = config.kubernetes.namespace;
+      name = "curl";
+    };
+    spec.containers = [{
+      name = "curl";
+      image = config.docker.images.curl.path;
+      args = ["curl" "--retry" "20" "--retry-connrefused" "http://nginx"];
+    }];
+    spec.restartPolicy = "Never";
+  });
+
 in {
   imports = [ kubenix.modules.test kubenix.modules.k8s kubenix.modules.docker ];
 
@@ -24,34 +40,55 @@ in {
       assertion = cfg.kind == "Deployment";
     } {
       message = "should have replicas set";
-      assertion = cfg.spec.replicas == 10;
+      assertion = cfg.spec.replicas == 3;
     }];
-    extraConfiguration = {
-      environment.systemPackages = [ pkgs.curl ];
-      services.kubernetes.kubelet.seedDockerImages = config.docker.export;
-    };
-    testScript = ''
-      kube.wait_until_succeeds("kubectl apply -f ${config.kubernetes.result}")
+    driver = "kubetest";
+    script = ''
+      import time
 
-      kube.succeed("kubectl get deployment | grep -i nginx")
-      kube.wait_until_succeeds("kubectl get deployment -o go-template nginx --template={{.status.readyReplicas}} | grep 10")
-      kube.wait_until_succeeds("curl http://nginx.default.svc.cluster.local | grep -i hello")
+      @pytest.mark.applymanifest('${config.kubernetes.resultYAML}')
+      def test_deployment(kube):
+          """Tests whether deployment gets successfully created"""
+
+          kube.wait_for_registered(timeout=30)
+
+          deployments = kube.get_deployments()
+          nginx_deploy = deployments.get('nginx')
+          assert nginx_deploy is not None
+
+          pods = nginx_deploy.get_pods()
+          assert len(pods) == 3
+
+          client_pod = kube.load_pod('${clientPod}')
+          client_pod.create()
+
+          client_pod.wait_until_ready(timeout=30)
+          client_pod.wait_until_containers_start()
+
+          container = client_pod.get_container('curl')
+
+          time.sleep(5)
+
+          logs = container.get_logs()
+
+          assert "Hello from NGINX" in logs
     '';
   };
 
-  docker.images.nginx.image = image;
-
-  kubernetes.version = k8sVersion;
+  docker.images = {
+    nginx.image = image;
+    curl.image = images.curl;
+  };
 
   kubernetes.resources.deployments.nginx = {
     spec = {
-      replicas = 10;
+      replicas = 3;
       selector.matchLabels.app = "nginx";
       template.metadata.labels.app = "nginx";
       template.spec = {
         containers.nginx = {
           image = config.docker.images.nginx.path;
-          imagePullPolicy = "Never";
+          imagePullPolicy = "IfNotPresent";
         };
       };
     };