WIP: test refactoring

modules/testing/default.nix

@@ -0,0 +1,99 @@
+{ nixosPath, config, pkgs, lib, kubenix, ... }:
+
+with lib;
+
+let
+  cfg = config.testing;
+
+  testModule = {
+    imports = [ ./test.nix ];
+
+    # passthru testing configuration
+    config._module.args = {
+      inherit pkgs kubenix;
+      testing = cfg;
+    };
+  };
+
+  isTestEnabled = test:
+    (cfg.enabledTests == null || elem test.name cfg.enabledTests) && test.enable;
+
+in {
+  imports = [
+    ./docker.nix
+    ./driver/kubetest.nix
+    ./runtime/local.nix
+  ];
+
+  options.testing = {
+    name = mkOption {
+      description = "Testing suite name";
+      type = types.str;
+      default = "default";
+    };
+
+    throwError = mkOption {
+      description = "Whether to throw error";
+      type = types.bool;
+      default = true;
+    };
+
+    defaults = mkOption {
+      description = "List of defaults to apply to tests";
+      type = types.listOf (types.submodule ({config, ...}: {
+        options = {
+          features = mkOption {
+            description = "List of features that test has to have to apply defaults";
+            type = types.listOf types.str;
+            default = [];
+          };
+
+          default = mkOption {
+            description = "Default to apply to test";
+            type = types.unspecified;
+            default = {};
+          };
+        };
+      }));
+      default = [];
+    };
+
+    tests = mkOption {
+      description = "List of test cases";
+      default = [];
+      type = types.listOf (types.coercedTo types.path (module: {
+        inherit module;
+      }) (types.submodule testModule));
+      apply = tests: filter isTestEnabled tests;
+    };
+
+    testsByName = mkOption {
+      description = "Tests by name";
+      type = types.attrsOf types.attrs;
+      default = listToAttrs (map (test: nameValuePair test.name test) cfg.tests);
+    };
+
+    enabledTests = mkOption {
+      description = "List of enabled tests (by default all tests are enabled)";
+      type = types.nullOr (types.listOf types.str);
+      default = null;
+    };
+
+    args = mkOption {
+      description = "Attribute set of extra args passed to tests";
+      type = types.attrs;
+      default = {};
+    };
+
+    success = mkOption {
+      description = "Whether testing was a success";
+      type = types.bool;
+      default = all (test: test.success) cfg.tests;
+    };
+
+    testScript = mkOption {
+      type = types.package;
+      description = "Script to run e2e tests";
+    };
+  };
+}

modules/testing/docker.nix

@@ -0,0 +1,46 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+with import ../../lib/docker.nix { inherit lib pkgs;  };
+
+let
+  testing = config.testing;
+
+  allImages = flatten (map (t: t.evaled.config.docker.export or []) testing.tests);
+
+  cfg = config.testing.docker;
+
+in {
+  options.testing.docker = {
+    registryUrl = mkOption {
+      description = "Docker registry url";
+      type = types.str;
+    };
+
+    images = mkOption {
+      description = "List of images to export";
+      type = types.listOf types.package;
+    };
+
+    copyScript = mkOption {
+      description = "Script to copy images to registry";
+      type = types.package;
+    };
+  };
+
+  config.testing.docker = {
+    images = allImages;
+
+    copyScript = copyDockerImages {
+      images = cfg.images;
+      dest = "docker://" + cfg.registryUrl;
+    };
+  };
+
+  config.testing.defaults = [{
+    features = ["docker"];
+    default = {
+      docker.registry.url = cfg.registryUrl;
+    };
+  }];
+}

modules/testing/driver/kubetest.nix

@@ -0,0 +1,51 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  testing = config.testing;
+  cfg = testing.driver.kubetest;
+
+  pythonEnv = pkgs.python37.withPackages (ps: with ps; [
+    pytest
+    kubetest
+    kubernetes
+  ] ++ cfg.extraPackages);
+
+  toTestScript = t:
+    if isString t.script
+    then pkgs.writeText "${t.name}.py" ''
+      ${cfg.defaultHeader}
+      ${t.script}
+    ''
+    else p.script;
+
+  tests = pkgs.linkFarm "${testing.name}-tests" (map (t: {
+    path = toTestScript t;
+    name = "${t.name}_test.py";
+  }) testing.tests);
+
+  testScript = pkgs.writeScript "test-${testing.name}.sh" ''
+    #!/usr/bin/env bash
+    ${pythonEnv}/bin/pytest -p no:cacheprovider ${tests} $@
+  '';
+
+in {
+  options.testing.driver.kubetest = {
+    defaultHeader = mkOption {
+      type = types.lines;
+      description = "Default test header";
+      default = ''
+        import pytest
+      '';
+    };
+
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      description = "Extra packages to pass to tests";
+      default = [];
+    };
+  };
+
+  config.testing.testScript = testScript;
+}

modules/testing/driver/kubetest/test-options.nix

@@ -0,0 +1,28 @@
+{ lib, config, ... }:
+
+with lib;
+
+let
+  cfg = config.kubetest;
+
+in {
+  options.test.kubetest = {
+    enable = mkOption {
+      description = "Whether to use kubetest test driver";
+      type = types.bool;
+      default = cfg.testScript != "";
+    };
+
+    testScript = mkOption {
+      type = types.lines;
+      description = "Test script to use for kubetest";
+      default = "";
+    };
+
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      description = "List of extra packages to use for kubetest";
+      default = [];
+    };
+  };
+}

modules/testing/driver/kubetest/testing-options.nix

@@ -0,0 +1,15 @@
+{ lib, config, ... }:
+
+with lib;
+
+{
+  options.testing.kubetest = {
+    defaultHeader = mkOption {
+      description = "Default test header";
+      type = types.lines;
+      default = ''
+        import pytest
+      '';
+    };
+  };
+}

modules/testing/runtime/local.nix

@@ -0,0 +1,65 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  testing = config.testing;
+
+  script = pkgs.writeScript "run-local-k8s-tests-${testing.name}.sh" ''
+    #!${pkgs.runtimeShell}
+
+    set -e
+
+    KUBECONFIG=''${KUBECONFIG:-~/.kube/config}
+    SKOPEOARGS=""
+
+    while (( "$#" )); do
+      case "$1" in
+        --kubeconfig)
+          KUBECONFIG=$2
+          shift 2
+          ;;
+        --skopeo-args)
+          SKOPEOARGS=$2
+          shift 2
+          ;;
+      esac
+    done
+
+    echo "--> copying docker images to registry"
+    ${testing.docker.copyScript} $SKOPEOARGS
+
+    echo "--> running tests"
+    ${testing.testScript} --kube-config=$KUBECONFIG
+  '';
+in {
+  options.testing.runtime.local = {
+    script = mkOption {
+      type = types.package;
+      description = "Runtime script";
+    };
+
+    docker = {
+      registryUrl = mkOption {
+        type = types.str;
+        description = "Docker registry url";
+      };
+
+      copyScript = mkOption {
+        type = types.package;
+        description = "Script used to copy docker images";
+      };
+    };
+    copyImages = mkOption {
+      type = types.package;
+      description = "Script used to copy docker images";
+    };
+
+    registryUrl = mkOption {
+      type = types.str;
+      description = "Registry url to copy docker images";
+    };
+  };
+
+  config.testing.runtime.local.script = script;
+}

modules/testing/runtime/nixos-k8s.nix

@@ -0,0 +1,130 @@
+# nixos-k8s implements nixos kubernetes testing runtime
+
+{ nixosPath, config, pkgs, lib, kubenix, ... }:
+
+let
+  testConfig = config.evaled.config;
+
+  nixosTesting = import "${nixosPath}/lib/testing.nix" {
+    inherit pkgs;
+    system = "x86_64-linux";
+  };
+
+  kubernetesBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
+    master = findFirst
+      (node: any (role: role == "master") node.config.services.kubernetes.roles)
+      (throw "no master node")
+      (attrValues nodes);
+    extraHosts = ''
+      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
+      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
+      ${concatMapStringsSep "\n"
+        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
+        (attrValues nodes)}
+    '';
+  in {
+    imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
+
+    config = mkMerge [
+      # base configuration for master and nodes
+      {
+        boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+        virtualisation.memorySize = mkDefault 2048;
+        virtualisation.cores = mkDefault 16;
+        virtualisation.diskSize = mkDefault 4096;
+        networking = {
+          inherit extraHosts;
+          domain = "my.xzy";
+          nameservers = ["10.0.0.254"];
+          firewall = {
+            allowedTCPPorts = [
+              10250 # kubelet
+            ];
+            trustedInterfaces = ["docker0" "cni0"];
+
+            extraCommands = concatMapStrings  (node: ''
+              iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+            '') (attrValues nodes);
+          };
+        };
+        environment.systemPackages = [ pkgs.kubectl ];
+        environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
+        services.flannel.iface = "eth1";
+        services.kubernetes = {
+          easyCerts = true;
+          apiserver = {
+            securePort = 443;
+            advertiseAddress = master.config.networking.primaryIPAddress;
+          };
+          masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
+          seedDockerImages = mkIf (elem "docker" testConfig._m.features) testConfig.docker.export;
+        };
+
+        systemd.extraConfig = "DefaultLimitNOFILE=1048576";
+      }
+
+      # configuration only applied on master nodes
+      (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
+        networking.firewall.allowedTCPPorts = [
+          443 # kubernetes apiserver
+        ];
+      })
+    ];
+  };
+
+  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
+    nixosTesting.makeTest {
+      inherit name;
+
+      nodes.kube = { config, pkgs, nodes, ... }: {
+        imports = [ kubernetesBaseConfig extraConfiguration ];
+        services.kubernetes = {
+          roles = ["master" "node"];
+          flannel.enable = false;
+          kubelet = {
+            networkPlugin = "cni";
+            cni.config = [{
+              name = "mynet";
+              type = "bridge";
+              bridge = "cni0";
+              addIf = true;
+              ipMasq = true;
+              isGateway = true;
+              ipam = {
+                type = "host-local";
+                subnet = "10.1.0.0/16";
+                gateway = "10.1.0.1";
+                routes = [{
+                  dst = "0.0.0.0/0";
+                }];
+              };
+            }];
+          };
+        };
+        networking.primaryIPAddress = mkForce "192.168.1.1";
+      };
+
+      testScript = ''
+        startAll;
+
+        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
+
+        ${testScript}
+      '';
+    };
+
+in {
+  options = {
+    runtime.nixos-k8s = {
+      driver = mkOption {
+        description = "Test driver";
+        type = types.package;
+        internal = true;
+      };
+    };
+
+    runtime.nixos-k8s.driver = mkKubernetesSingleNodeTest {
+      inherit (config) name testScript;
+    };
+  };
+}

modules/testing/runtime/nixos-k8s/runtime-options.nix

@@ -0,0 +1,7 @@
+{ config, ... }:
+
+{
+  options.runtime.nixos-k8s = {
+
+  };
+}

modules/testing/test-options.nix

@@ -0,0 +1,72 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.test;
+
+in {
+  options.test = {
+    name = mkOption {
+      description = "Test name";
+      type = types.str;
+    };
+
+    description = mkOption {
+      description = "Test description";
+      type = types.str;
+    };
+
+    enable = mkOption {
+      description = "Whether to enable test";
+      type = types.bool;
+      default = true;
+    };
+
+    assertions = mkOption {
+      type = types.listOf (types.submodule {
+        options = {
+          assertion = mkOption {
+            description = "assertion value";
+            type = types.bool;
+            default = false;
+          };
+
+          message = mkOption {
+            description = "assertion message";
+            type = types.str;
+          };
+        };
+      });
+      default = [];
+      example = [ { assertion = false; message = "you can't enable this for some reason"; } ];
+      description = ''
+        This option allows modules to express conditions that must
+        hold for the evaluation of the system configuration to
+        succeed, along with associated error messages for the user.
+      '';
+    };
+
+    script = mkOption {
+      description = "Test script to use for e2e test";
+      type = types.nullOr (types.either types.lines types.path);
+    };
+
+    testScript = mkOption {
+      description = "Script to run as part of testing";
+      type = types.nullOr types.lines;
+      default = null;
+    };
+
+    distro = mkOption {
+      description = "Kubernetes distro to run the test with. Defaults to 'nixos', other option is 'k3s'";
+      type = types.nullOr types.str;
+      default = null;
+    };
+
+    driver = mkOption {
+      description = "Name of the driver to use for testing";
+      type = types.str;
+    };
+  };
+}

modules/testing/test.nix

@@ -0,0 +1,134 @@
+{ lib, config, testing, kubenix, ... }:
+
+with lib;
+
+let
+  modules = [
+    # testing module
+    config.module
+
+    ./test-options.nix
+    ../base.nix
+
+    # passthru some options to test
+    {
+      config = {
+        kubenix.project = mkDefault config.name;
+        _module.args = {
+          inherit kubenix;
+          test = config;
+        } // testing.args;
+      };
+    }
+  ];
+
+  # eval without checking
+  evaled' = kubenix.evalModules {
+    check = false;
+    inherit modules;
+  };
+
+  # test configuration
+  testConfig = evaled'.config.test;
+
+  # test features
+  testFeatures = evaled'.config._m.features;
+
+  # defaults that can be applied on tests
+  defaults =
+    filter (d:
+      (intersectLists d.features testFeatures) == d.features ||
+      (length d.features) == 0
+    ) testing.defaults;
+
+  # add default modules to all modules
+  modulesWithDefaults = modules ++ (map (d: d.default) defaults);
+
+  # evaled test
+  evaled = let
+    evaled' = kubenix.evalModules {
+      modules = modulesWithDefaults;
+    };
+  in
+    if testing.throwError then evaled'
+    else if (builtins.tryEval evaled'.config.test.assertions).success
+    then evaled' else null;
+
+in {
+  imports = [
+    ./driver/kubetest.nix
+  ];
+
+  options = {
+    name = mkOption {
+      description = "test name";
+      type = types.str;
+      internal = true;
+    };
+
+    description = mkOption {
+      description = "test description";
+      type = types.str;
+      internal = true;
+    };
+
+    enable = mkOption {
+      description = "Whether to enable test";
+      type = types.bool;
+      internal = true;
+    };
+
+    module = mkOption {
+      description = "Module defining kubenix test";
+      type = types.unspecified;
+    };
+
+    evaled = mkOption {
+      description = "Test evaulation result";
+      type = types.nullOr types.attrs;
+      internal = true;
+    };
+
+    success = mkOption {
+      description = "Whether test assertions were successfull";
+      type = types.bool;
+      internal = true;
+      default = false;
+    };
+
+    assertions = mkOption {
+      description = "Test result";
+      type = types.unspecified;
+      internal = true;
+      default = [];
+    };
+
+    script = mkOption {
+      description = "Test script to use for e2e test";
+      type = types.nullOr (types.either types.lines types.path);
+      internal = true;
+    };
+
+    driver = mkOption {
+      description = "Name of the driver to use for testing";
+      type = types.str;
+      internal = true;
+    };
+  };
+
+  config = mkMerge [
+    {
+      inherit evaled;
+      inherit (testConfig) name description enable driver;
+    }
+
+    # if test is evaled check assertions
+    (mkIf (config.evaled != null) {
+      inherit (evaled.config.test) assertions;
+
+      # if all assertions are true, test is successfull
+      success = all (el: el.assertion) config.assertions;
+      script = evaled.config.test.script;
+    })
+  ];
+}