fixup tests

modules/testing/default.nix

@@ -0,0 +1,104 @@
+{ config, pkgs, lib, kubenix, ... }:
+
+with lib;
+let
+  cfg = config.testing;
+
+  testModule = {
+    imports = [ ./evalTest.nix ];
+
+    # passthru testing configuration
+    config._module.args = {
+      inherit pkgs kubenix;
+      testing = cfg;
+    };
+  };
+
+  isTestEnabled = test:
+    (cfg.enabledTests == null || elem test.name cfg.enabledTests) && test.enable;
+
+in
+{
+  imports = [
+    ./docker.nix
+    ./driver/kubetest.nix
+    ./runtime/local.nix
+    ./runtime/nixos-k8s.nix
+  ];
+
+  options.testing = {
+    name = mkOption {
+      description = "Testing suite name";
+      type = types.str;
+      default = "default";
+    };
+
+    throwError = mkOption {
+      description = "Whether to throw error";
+      type = types.bool;
+      default = true;
+    };
+
+    defaults = mkOption {
+      description = "List of defaults to apply to tests";
+      type = types.listOf (types.submodule ({ config, ... }: {
+        options = {
+          features = mkOption {
+            description = "List of features that test has to have to apply defaults";
+            type = types.listOf types.str;
+            default = [ ];
+          };
+
+          default = mkOption {
+            description = "Default to apply to test";
+            type = types.unspecified;
+            default = { };
+          };
+        };
+      }));
+      default = [ ];
+    };
+
+    tests = mkOption {
+      description = "List of test cases";
+      default = [ ];
+      type = types.listOf (types.coercedTo types.path
+        (module: {
+          inherit module;
+        })
+        (types.submodule testModule));
+      apply = tests: filter isTestEnabled tests;
+    };
+
+    testsByName = mkOption {
+      description = "Tests by name";
+      type = types.attrsOf types.attrs;
+      default = listToAttrs (map (test: nameValuePair test.name test) cfg.tests);
+    };
+
+    enabledTests = mkOption {
+      description = "List of enabled tests (by default all tests are enabled)";
+      type = types.nullOr (types.listOf types.str);
+      default = null;
+    };
+
+    args = mkOption {
+      description = "Attribute set of extra args passed to tests";
+      type = types.attrs;
+      default = { };
+    };
+
+    success = mkOption {
+      internal = true; # read only property
+      description = "Whether testing was a success";
+      type = types.bool;
+      default = all (test: test.success) cfg.tests;
+    };
+
+    testScript = mkOption {
+      internal = true; # set by test driver
+      type = types.package;
+      description = "Script to run e2e tests";
+    };
+  };
+}

modules/testing/docker.nix

@@ -0,0 +1,46 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+with import ../../lib/docker { inherit lib pkgs; };
+let
+  testing = config.testing;
+
+  allImages = unique (flatten (map (t: t.evaled.config.docker.export or [ ]) testing.tests));
+
+  cfg = config.testing.docker;
+
+in
+{
+  options.testing.docker = {
+    registryUrl = mkOption {
+      description = "Docker registry url";
+      type = types.str;
+    };
+
+    images = mkOption {
+      description = "List of images to export";
+      type = types.listOf types.package;
+    };
+
+    copyScript = mkOption {
+      description = "Script to copy images to registry";
+      type = types.package;
+    };
+  };
+
+  config.testing.docker = {
+    images = allImages;
+
+    copyScript = copyDockerImages {
+      images = cfg.images;
+      dest = "docker://" + cfg.registryUrl;
+    };
+  };
+
+  config.testing.defaults = [{
+    features = [ "docker" ];
+    default = {
+      docker.registry.url = cfg.registryUrl;
+    };
+  }];
+}

modules/testing/driver/kubetest.nix

@@ -0,0 +1,58 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+let
+  testing = config.testing;
+  cfg = testing.driver.kubetest;
+
+  kubetest = import ./kubetestdrv.nix { inherit pkgs; };
+
+  pythonEnv = pkgs.python38.withPackages (ps: with ps; [
+    pytest
+    kubetest
+    kubernetes
+  ] ++ cfg.extraPackages);
+
+  toTestScript = t:
+    if isString t.script
+    then
+      pkgs.writeText "${t.name}.py" ''
+        ${cfg.defaultHeader}
+        ${t.script}
+      ''
+    else t.script;
+
+  tests = pkgs.linkFarm "${testing.name}-tests" (
+    map
+      (t: {
+        path = toTestScript t;
+        name = "${t.name}_test.py";
+      })
+      (filter (t: t.script != null) testing.tests)
+  );
+
+  testScript = pkgs.writeScript "test-${testing.name}.sh" ''
+    #!/usr/bin/env bash
+    ${pythonEnv}/bin/pytest -p no:cacheprovider ${tests} $@
+  '';
+
+in
+{
+  options.testing.driver.kubetest = {
+    defaultHeader = mkOption {
+      type = types.lines;
+      description = "Default test header";
+      default = ''
+        import pytest
+      '';
+    };
+
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      description = "Extra packages to pass to tests";
+      default = [ ];
+    };
+  };
+
+  config.testing.testScript = testScript;
+}

modules/testing/driver/kubetestdrv.nix

@@ -0,0 +1,15 @@
+{ pkgs ? import <nixpkgs> { } }:
+with pkgs;
+with pkgs.python38Packages;
+
+with pkgs.python38;
+pkgs.python38Packages.buildPythonPackage rec {
+  pname = "kubetest";
+  version = "0.9.5";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-TqDHMciAEXv4vMWLJY1YdtXsP4ho+INgdFB3xQQNoZU=";
+  };
+  propagatedBuildInputs = [ pytest kubernetes ];
+  doCheck = false;
+}

modules/testing/evalTest.nix

@@ -0,0 +1,128 @@
+{ lib, config, testing, kubenix, ... }:
+
+with lib;
+let
+  modules = [
+    # testing module
+    config.module
+
+    ./test-options.nix
+    ../base.nix
+
+    # passthru some options to test
+    {
+      config = {
+        kubenix.project = mkDefault config.name;
+        _module.args = {
+          inherit kubenix;
+          test = config;
+        } // testing.args;
+      };
+    }
+  ];
+
+  # eval without checking
+  evaled' = kubenix.evalModules {
+    check = false;
+    inherit modules;
+  };
+
+  # test configuration
+  testConfig = evaled'.config.test;
+
+  # test features
+  testFeatures = evaled'.config._m.features;
+
+  # defaults that can be applied on tests
+  defaults =
+    filter
+      (d:
+        (intersectLists d.features testFeatures) == d.features ||
+        (length d.features) == 0
+      )
+      testing.defaults;
+
+  # add default modules to all modules
+  modulesWithDefaults = modules ++ (map (d: d.default) defaults);
+
+  # evaled test
+  evaled =
+    let
+      evaled' = kubenix.evalModules {
+        modules = modulesWithDefaults;
+      };
+    in
+    if testing.throwError then evaled'
+    else if (builtins.tryEval evaled'.config.test.assertions).success
+    then evaled' else null;
+
+in
+{
+  options = {
+    module = mkOption {
+      description = "Module defining kubenix test";
+      type = types.unspecified;
+    };
+
+    evaled = mkOption {
+      description = "Test evaulation result";
+      type = types.nullOr types.attrs;
+      internal = true;
+    };
+
+    success = mkOption {
+      description = "Whether test assertions were successfull";
+      type = types.bool;
+      internal = true;
+      default = false;
+    };
+
+    # transparently forwarded from the test's `test` attribute for ease of access
+    name = mkOption {
+      description = "test name";
+      type = types.str;
+      internal = true;
+    };
+
+    description = mkOption {
+      description = "test description";
+      type = types.str;
+      internal = true;
+    };
+
+    enable = mkOption {
+      description = "Whether to enable test";
+      type = types.bool;
+      internal = true;
+    };
+
+    assertions = mkOption {
+      description = "Test result";
+      type = types.unspecified;
+      internal = true;
+      default = [ ];
+    };
+
+    script = mkOption {
+      description = "Test script to use for e2e test";
+      type = types.nullOr (types.either types.lines types.path);
+      internal = true;
+    };
+
+  };
+
+  config = mkMerge [
+    {
+      inherit evaled;
+      inherit (testConfig) name description enable;
+    }
+
+    # if test is evaled check assertions
+    (mkIf (config.evaled != null) {
+      inherit (evaled.config.test) assertions script;
+
+      # if all assertions are true, test is successfull
+      success = all (el: el.assertion) config.assertions;
+    })
+  ];
+}

modules/testing/runtime/local.nix

@@ -0,0 +1,44 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+let
+  testing = config.testing;
+
+  script = pkgs.writeScript "run-local-k8s-tests-${testing.name}.sh" ''
+    #!${pkgs.runtimeShell}
+
+    set -e
+
+    KUBECONFIG=''${KUBECONFIG:-~/.kube/config}
+    SKOPEOARGS=""
+
+    while (( "$#" )); do
+      case "$1" in
+        --kubeconfig)
+          KUBECONFIG=$2
+          shift 2
+          ;;
+        --skopeo-args)
+          SKOPEOARGS=$2
+          shift 2
+          ;;
+      esac
+    done
+
+    echo "--> copying docker images to registry"
+    ${testing.docker.copyScript} $SKOPEOARGS
+
+    echo "--> running tests"
+    ${testing.testScript} --kube-config=$KUBECONFIG
+  '';
+in
+{
+  options.testing.runtime.local = {
+    script = mkOption {
+      type = types.package;
+      description = "Runtime script";
+    };
+  };
+
+  config.testing.runtime.local.script = script;
+}

modules/testing/runtime/nixos-k8s.nix

@@ -0,0 +1,95 @@
+# nixos-k8s implements nixos kubernetes testing runtime
+
+{ config
+, pkgs
+, lib
+, ...
+}:
+
+with lib;
+let
+  testing = config.testing;
+  # kubeconfig = "/etc/${config.services.kubernetes.pki.etcClusterAdminKubeconfig}";
+  kubeconfig = "/etc/kubernetes/cluster-admin.kubeconfig";
+  kubecerts = "/var/lib/kubernetes/secrets";
+
+  # how we differ from the standard configuration of mkKubernetesBaseTest
+  extraConfiguration = { config, pkgs, lib, nodes, ... }: {
+
+    virtualisation = {
+      memorySize = 2048;
+    };
+
+    networking = {
+      nameservers = [ "10.0.0.254" ];
+      firewall = {
+        trustedInterfaces = [ "docker0" "cni0" ];
+      };
+    };
+
+    services.kubernetes = {
+      flannel.enable = false;
+      kubelet = {
+        seedDockerImages = testing.docker.images;
+        networkPlugin = "cni";
+        cni.config = [{
+          name = "mynet";
+          type = "bridge";
+          bridge = "cni0";
+          addIf = true;
+          ipMasq = true;
+          isGateway = true;
+          ipam = {
+            type = "host-local";
+            subnet = "10.1.0.0/16";
+            gateway = "10.1.0.1";
+            routes = [{
+              dst = "0.0.0.0/0";
+            }];
+          };
+        }];
+      };
+    };
+
+    systemd = {
+      extraConfig = "DefaultLimitNOFILE=1048576";
+      # Host tools should have a chance to access guest's kube api
+      services.copy-certs = {
+        description = "Share k8s certificates with host";
+        script = "cp -rf ${kubecerts} /tmp/xchg/; cp -f ${kubeconfig} /tmp/xchg/;";
+        after = [ "kubernetes.target" ];
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+      };
+    };
+
+  };
+
+  script = ''
+    machine1.succeed("${testing.testScript} --kube-config=${kubeconfig}")
+  '';
+
+  test =
+    with import "${pkgs.path}/nixos/tests/kubernetes/base.nix" { inherit pkgs; inherit (pkgs) system; };
+    mkKubernetesSingleNodeTest {
+      inherit extraConfiguration;
+      inherit (config.testing) name;
+      test = script;
+    };
+
+
+in
+{
+  options.testing.runtime.nixos-k8s = {
+    driver = mkOption {
+      description = "Test driver";
+      type = types.package;
+      internal = true;
+    };
+  };
+
+  config.testing.runtime.nixos-k8s.driver = test.driver;
+}

modules/testing/test-options.nix

@@ -0,0 +1,57 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.test;
+
+in
+{
+  options.test = {
+    name = mkOption {
+      description = "Test name";
+      type = types.str;
+    };
+
+    description = mkOption {
+      description = "Test description";
+      type = types.str;
+    };
+
+    enable = mkOption {
+      description = "Whether to enable test";
+      type = types.bool;
+      default = true;
+    };
+
+    assertions = mkOption {
+      type = types.listOf (types.submodule {
+        options = {
+          assertion = mkOption {
+            description = "assertion value";
+            type = types.bool;
+            default = false;
+          };
+
+          message = mkOption {
+            description = "assertion message";
+            type = types.str;
+          };
+        };
+      });
+      default = [ ];
+      example = [{ assertion = false; message = "you can't enable this for some reason"; }];
+      description = ''
+        This option allows modules to express conditions that must
+        hold for the evaluation of the system configuration to
+        succeed, along with associated error messages for the user.
+      '';
+    };
+
+    script = mkOption {
+      description = "Test script to use for e2e test";
+      type = types.nullOr (types.either types.lines types.path);
+      default = null;
+    };
+
+  };
+}