refator: put all modules under modules directory

modules/default.nix

@@ -0,0 +1,8 @@
+{
+  submodules = ./submodules.nix;
+  k8s = ./k8s;
+  k8s-submodules = ./k8s/submodule.nix;
+  istio = ./istio;
+  testing = ./testing;
+  helm = ./helm;
+}

modules/helm/chart2json.nix

@@ -0,0 +1,51 @@
+{ stdenvNoCC, lib, kubernetes-helm, gawk, remarshal, jq }:
+
+with lib;
+
+{
+  # chart to template
+  chart
+
+  # release name
+, name
+
+  # namespace to install release into
+, namespace ? null
+
+  # values to pass to chart
+, values ? {}
+
+  # kubernetes version to template chart for
+, kubeVersion ? null }: let
+  valuesJsonFile = builtins.toFile "${name}-values.json" (builtins.toJSON values);
+in stdenvNoCC.mkDerivation {
+  name = "${name}.json";
+  buildCommand = ''
+    # template helm file and write resources to yaml
+    helm template --name "${name}" \
+      ${optionalString (kubeVersion != null) "--kube-version ${kubeVersion}"} \
+      ${optionalString (namespace != null) "--namespace ${namespace}"} \
+      ${optionalString (values != {}) "-f ${valuesJsonFile}"} \
+      ${chart} >resources.yaml
+
+    # split multy yaml file into multiple files
+    awk 'BEGIN{i=1}{line[i++]=$0}END{j=1;n=0; while (j<i) {if (line[j] ~ /^---/) n++; else print line[j] >>"resource-"n".yaml"; j++}}' resources.yaml
+
+    # join multiple yaml files in jsonl file
+    for file in ./resource-*.yaml
+    do
+      remarshal -i $file -if yaml -of json >>resources.jsonl
+    done
+
+    # convert jsonl file to json array, remove null values and write to $out
+    cat resources.jsonl | jq -Scs 'walk(
+      if type == "object" then
+        with_entries(select(.value != null))
+      elif type == "array" then
+        map(select(. != null))
+      else
+        .
+      end)' > $out
+  '';
+  nativeBuildInputs = [ kubernetes-helm gawk remarshal jq ];
+}

modules/helm/default.nix

@@ -0,0 +1,112 @@
+{ config, lib, pkgs, kubenix, ... }:
+
+with lib;
+
+let
+  cfg = config.kubernetes.helm;
+
+  globalConfig = config;
+
+  recursiveAttrs = mkOptionType {
+    name = "recursive-attrs";
+    description = "recursive attribute set";
+    check = isAttrs;
+    merge = loc: foldl' (res: def: recursiveUpdate res def.value) {};
+  };
+
+  parseApiVersion = apiVersion: let
+    splitted = splitString "/" apiVersion;
+  in {
+    group = if length splitted == 1 then "core" else head splitted;
+    version = last splitted;
+  };
+
+  chart2json = pkgs.callPackage ./chart2json.nix {  };
+  fetchhelm = pkgs.callPackage ./fetchhelm.nix {  };
+
+in {
+  imports = [
+    kubenix.k8s
+  ];
+
+  options.kubernetes.helm = {
+    instances = mkOption {
+      description = "Attribute set of helm instances";
+      type = types.attrsOf (types.submodule ({ config, name, ... }: {
+        options = {
+          name = mkOption {
+            description = "Helm release name";
+            type = types.str;
+            default = name;
+          };
+
+          chart = mkOption {
+            description = "Helm chart to use";
+            type = types.package;
+          };
+
+          namespace = mkOption {
+            description = "Namespace to install helm chart to";
+            type = types.nullOr types.str;
+            default = null;
+          };
+
+          values = mkOption {
+            description = "Values to pass to chart";
+            type = recursiveAttrs;
+            default = {};
+          };
+
+          kubeVersion = mkOption {
+            description = "Kubernetes version to build chart for";
+            type = types.str;
+            default = globalConfig.kubernetes.version;
+          };
+
+          overrides = mkOption {
+            description = "Overrides to apply to all chart resources";
+            type = types.listOf types.unspecified;
+            default = [];
+          };
+
+          overrideNamespace = mkOption {
+            description = "Whether to apply namespace override";
+            type = types.bool;
+            default = true;
+          };
+
+          objects = mkOption {
+            description = "Generated kubernetes objects";
+            type = types.listOf types.attrs;
+            default = [];
+          };
+        };
+
+        config.overrides = mkIf (config.overrideNamespace && config.namespace != null) [{
+          metadata.namespace = mkDefault config.namespace;
+        }];
+
+        config.objects = importJSON (chart2json {
+          inherit (config) chart name namespace values kubeVersion;
+        });
+      }));
+    };
+  };
+
+  # include helper helm methods as args
+  config._module.args.helm = {
+    fetch = fetchhelm;
+    chart2json = chart2json;
+  };
+
+  config.kubernetes.api = mkMerge (flatten (mapAttrsToList (_: instance:
+    map (object: let
+      apiVersion = parseApiVersion object.apiVersion;
+      name = object.metadata.name;
+    in {
+      "${apiVersion.group}"."${apiVersion.version}".${object.kind}."${name}" = mkMerge ([
+        object
+      ] ++ instance.overrides);
+    }) instance.objects
+  ) cfg.instances));
+}

modules/helm/fetchhelm.nix

@@ -0,0 +1,48 @@
+{ stdenvNoCC, lib, kubernetes-helm, cacert }:
+
+let
+  cleanName = name: lib.replaceStrings ["/"] ["-"] name;
+
+in {
+  # name of the chart
+  chart
+
+  # chart url to fetch from custom location
+, chartUrl ? null
+
+  # version of the chart
+, version ? null
+
+# chart hash
+, sha256
+
+# whether to extract chart
+, untar ? true
+
+# use custom charts repo
+, repo ? null
+
+# pass --verify to helm chart
+, verify ? false
+
+# pass --devel to helm chart
+, devel ? false }: stdenvNoCC.mkDerivation {
+  name = "${cleanName chart}-${if version == null then "dev" else version}";
+
+  buildCommand = ''
+    export HOME="$PWD"
+    helm init --client-only >/dev/null
+    ${if repo == null then "" else "helm repo add repository ${repo}"}
+    helm fetch -d ./chart \
+      ${if untar then "--untar" else ""} \
+      ${if version == null then "" else "--version ${version}"} \
+      ${if devel then "--devel" else ""} \
+      ${if verify then "--verify" else ""} \
+      ${if chartUrl == null then (if repo == null then chart else "repository/${chart}") else chartUrl}
+    cp -r chart/*/ $out
+  '';
+  outputHashMode = "recursive";
+  outputHashAlgo = "sha256";
+  outputHash = sha256;
+  nativeBuildInputs = [ kubernetes-helm cacert ];
+}

modules/helm/test.nix

@@ -0,0 +1,43 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+let
+  fetchhelm = pkgs.callPackage ./fetchhelm.nix {  };
+  chart2json = pkgs.callPackage ./chart2json.nix {  };
+in rec {
+  postgresql-chart = fetchhelm {
+    chart = "stable/postgresql";
+    version = "0.18.1";
+    sha256 = "1p3gfmaakxrqb4ncj6nclyfr5afv7xvcdw95c6qyazfg72h3zwjn";
+  };
+
+  istio-chart = fetchhelm {
+    chart = "istio";
+    version = "1.1.0";
+    repo = "https://storage.googleapis.com/istio-release/releases/1.1.0-rc.0/charts";
+    sha256 = "0ippv2914hwpsb3kkhk8d839dii5whgrhxjwhpb9vdwgji5s7yfl";
+  };
+
+  istio-official-chart = pkgs.fetchgit {
+    url = "https://github.com/fyery-chen/istio-helm";
+    rev = "47e235e775314daeb88a3a53689ed66c396ecd3f";
+    sha256 = "190sfyvhdskw6ijy8cprp6hxaazn7s7mg5ids4snshk1pfdg2q8h";
+  };
+
+  postgresql-json = chart2json {
+    name = "postgresql";
+    chart = postgresql-chart;
+    values = {
+      networkPolicy.enabled = true;
+    };
+  };
+
+  istio-json = chart2json {
+    name = "istio";
+    chart = istio-chart;
+  };
+
+  istio-official-json = chart2json {
+    name = "istio-official";
+    chart = "${istio-official-chart}/istio-official";
+  };
+}

modules/istio/overrides.nix

@@ -0,0 +1,15 @@
+{ lib, definitions }:
+
+with lib;
+
+{
+  "istio_networking_v1alpha3_StringMatch" = recursiveUpdate (recursiveUpdate
+    definitions."istio_networking_v1alpha3_StringMatch_Exact"
+    definitions."istio_networking_v1alpha3_StringMatch_Prefix"
+  )
+  definitions."istio_networking_v1alpha3_StringMatch_Regex";
+
+  "istio_networking_v1alpha3_PortSelector" = recursiveUpdate
+    definitions."istio_networking_v1alpha3_PortSelector_Name"
+    definitions."istio_networking_v1alpha3_PortSelector_Number";
+}

modules/k8s/default.nix

@@ -0,0 +1,263 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.kubernetes;
+
+  removeKubenixOptions = filterAttrs (name: attr: name != "kubenix");
+
+  getDefaults = resource: group: version: kind:
+    catAttrs "default" (filter (default:
+      (resource == null || default.resource == null || default.resource == resource) &&
+      (default.group == null || default.group == group) &&
+      (default.version == null || default.version == version) &&
+      (default.kind == null || default.kind == kind)
+    ) cfg.api.defaults);
+
+  moduleToAttrs = value:
+    if isAttrs value
+    then mapAttrs (n: v: moduleToAttrs v) (filterAttrs (n: v: !(hasPrefix "_" n) && v != null) value)
+
+    else if isList value
+    then map (v: moduleToAttrs v) value
+
+    else value;
+
+  flattenResources = resources: flatten (
+    mapAttrsToList (groupName: versions:
+      mapAttrsToList (versionName: kinds:
+        builtins.trace versionName kinds
+      ) versions
+    ) resources
+  );
+
+  toKubernetesList = resources: {
+    kind = "List";
+    apiVersion = "v1";
+    items = resources;
+  };
+
+  apiOptions = { config, ... }: {
+    options = {
+      definitions = mkOption {
+        description = "Attribute set of kubernetes definitions";
+      };
+
+      defaults = mkOption {
+        description = "Kubernetes defaults to apply to resources";
+        type = types.listOf (types.submodule ({config, ...}: {
+          options = {
+            resource = mkOption {
+              description = "Resource to apply default to (all by default)";
+              type = types.nullOr types.str;
+              default = null;
+            };
+
+            group = mkOption {
+              description = "Group to apply default to (all by default)";
+              type = types.nullOr types.str;
+              default = null;
+            };
+
+            version = mkOption {
+              description = "Version to apply default to (all by default)";
+              type = types.nullOr types.str;
+              default = null;
+            };
+
+            kind = mkOption {
+              description = "Kind to apply default to (all by default)";
+              type = types.nullOr types.str;
+              default = null;
+            };
+
+            default = mkOption {
+              description = "Default to apply";
+              type = types.unspecified;
+              default = {};
+            };
+          };
+        }));
+        default = [];
+      };
+
+      resources = mkOption {
+        type = types.listOf (types.submodule {
+          options = {
+            group = mkOption {
+              description = "Resoruce group";
+              type = types.str;
+            };
+
+            version = mkOption {
+              description = "Resoruce version";
+              type = types.str;
+            };
+
+            kind = mkOption {
+              description = "Resource kind";
+              type = types.str;
+            };
+
+            resource = mkOption {
+              description = "Resource name";
+              type = type.str;
+            };
+          };
+        });
+        default = [];
+      };
+    };
+  };
+
+  indexOf = lst: value:
+    head (filter (v: v != -1) (imap0 (i: v: if v == value then i else -1) lst));
+in {
+  imports = [./lib.nix];
+
+  options.kubernetes.version = mkOption {
+    description = "Kubernetes version to use";
+    type = types.enum ["1.7" "1.8" "1.9" "1.10" "1.11" "1.12" "1.13"];
+    default = "1.13";
+  };
+
+  options.kubernetes.resourceOrder = mkOption {
+    description = "Preffered resource order";
+    type = types.listOf types.str;
+    default = [
+      "CustomResourceDefinition"
+      "Namespace"
+    ];
+  };
+
+  options.kubernetes.api = mkOption {
+    type = types.submodule {
+      imports = [
+        (./generated + ''/v'' + cfg.version + ".nix")
+        apiOptions
+      ] ++ (map (cr: {config, ...}: {
+        options.${cr.group}.${cr.version}.${cr.kind} = mkOption {
+          description = cr.description;
+          type = types.attrsOf (types.submodule ({name, ...}: {
+            imports = getDefaults cr.resource cr.group cr.version cr.kind;
+            options = {
+              apiVersion = mkOption {
+                description = "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources";
+                type = types.nullOr types.str;
+              };
+
+              kind = mkOption {
+                description = "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds";
+                type = types.nullOr types.str;
+              };
+
+              metadata = mkOption {
+                description = "Standard object metadata; More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata.";
+                type = types.nullOr (types.submodule config.definitions."io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta");
+              };
+
+              spec = mkOption {
+                description = "Module spec";
+                type = types.submodule cr.module;
+                default = {};
+              };
+            };
+
+            config = {
+              apiVersion = mkOptionDefault "${cr.group}/${cr.version}";
+              kind = mkOptionDefault cr.kind;
+              metadata.name = mkOptionDefault name;
+            };
+          }));
+          default = {};
+        };
+      }) cfg.customResources);
+    };
+    default = {};
+  };
+
+  options.kubernetes.customResources = mkOption {
+    default = [];
+    description = "List of custom resource definitions to make API for";
+    type = types.listOf (types.submodule ({config, ...}: {
+      options = {
+        group = mkOption {
+          description = "Custom resource definition group";
+          type = types.str;
+        };
+
+        version = mkOption {
+          description = "Custom resource definition version";
+          type = types.str;
+        };
+
+        kind = mkOption {
+          description = "Custom resource definition kind";
+          type = types.str;
+        };
+
+        resource = mkOption {
+          description = "Custom resource definition resource name";
+          type = types.nullOr types.str;
+          default = null;
+        };
+
+        description = mkOption {
+          description = "Custom resource definition description";
+          type = types.str;
+          default = "";
+        };
+
+        module = mkOption {
+          description = "Custom resource definition module";
+          default = types.unspecified;
+        };
+      };
+    }));
+  };
+
+  config.kubernetes.api.resources = map (cr: {
+    inherit (cr) group version kind resource;
+  }) cfg.customResources;
+
+  options.kubernetes.objects = mkOption {
+    description = "Attribute set of kubernetes objects";
+    type = types.listOf types.attrs;
+    apply = items: sort (r1: r2:
+      if elem r1.kind cfg.resourceOrder && elem r2.kind cfg.resourceOrder
+      then indexOf cfg.resourceOrder r1.kind < indexOf cfg.resourceOrder r2.kind
+      else if elem r1.kind cfg.resourceOrder then true else false
+    ) (moduleToAttrs (unique items));
+    default = [];
+  };
+
+  config.kubernetes.objects = flatten (map (gvk:
+    mapAttrsToList (name: resource:
+      removeKubenixOptions (moduleToAttrs resource)
+    ) cfg.api.${gvk.group}.${gvk.version}.${gvk.kind}
+  ) cfg.api.resources);
+
+  options.kubernetes.generated = mkOption {
+    type = types.attrs;
+    description = "Generated json file";
+  };
+
+  config.kubernetes.generated = let
+    kubernetesList = toKubernetesList cfg.objects;
+
+    hashedList = kubernetesList // {
+      labels."kubenix/build" = cfg.hash;
+      items = map (resource: recursiveUpdate resource {
+        metadata.labels."kubenix/build" = cfg.hash;
+      }) kubernetesList.items;
+    };
+  in hashedList;
+
+  options.kubernetes.hash = mkOption {
+    type = types.str;
+    description = "Output hash";
+  };
+
+  config.kubernetes.hash = builtins.hashString "sha1" (builtins.toJSON cfg.objects);
+}

modules/k8s/lib.nix

@@ -0,0 +1,35 @@
+{ lib, ... }:
+
+with lib;
+
+let
+  k8s = {
+    mkSecretOption = {description ? "", default ? null}: mkOption {
+      inherit description;
+      type = types.nullOr (types.submodule {
+        options = {
+          name = mkOption {
+            description = "Name of the secret where secret is stored";
+            type = types.str;
+          };
+
+          key = mkOption {
+            description = "Name of the key where secret is stored";
+            type = types.str;
+          };
+        };
+
+        config = mkDefault (if default == null then {} else default);
+      });
+      default = {};
+    };
+
+    secretToEnv = value: {
+      valueFrom.secretKeyRef = {
+        inherit (value) name key;
+      };
+    };
+  };
+in {
+  _module.args.k8s = k8s;
+}

modules/k8s/submodule.nix

@@ -0,0 +1,59 @@
+{ config, lib, kubenix, ... }:
+
+with lib;
+
+let
+  globalConfig = config;
+in {
+  imports = [ kubenix.submodules ];
+
+  options = {
+    kubernetes.propagateDefaults = mkOption {
+      description = "Whether to propagate child defaults to submodules";
+      type = types.bool;
+      default = true;
+    };
+
+    submodules.instances = mkOption {
+       type = types.attrsOf (types.submodule ({config, ...}: {
+        options = {
+          namespace = mkOption {
+            description = "Default kubernetes namespace";
+            type = types.str;
+            default = "default";
+          };
+        };
+
+        config.config = {
+          kubernetes.api.defaults = [{
+            default.metadata.namespace = mkDefault config.namespace;
+          }];
+        };
+      }));
+    };
+  };
+
+  config = {
+    submodules.defaults = [{
+      default = {
+        imports = [ kubenix.k8s ];
+        kubernetes.version = mkDefault config.kubernetes.version;
+        kubernetes.api.defaults =
+          mkIf config.kubernetes.propagateDefaults config.kubernetes.api.defaults;
+      };
+    } {
+      default = ({config, ...}: {
+        kubernetes.api.defaults = [{
+          default.metadata.labels = {
+            "kubenix/module-name" = config.submodule.name;
+            "kubenix/module-version" = config.submodule.version;
+          };
+        }];
+      });
+    }];
+
+    kubernetes.objects = mkMerge (mapAttrsToList (_: submodule:
+      submodule.config.kubernetes.objects
+    ) config.submodules.instances);
+  };
+}

modules/submodules.nix

@@ -0,0 +1,256 @@
+{ config, kubenix, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.submodules;
+
+  getDefaults = name: tags:
+    catAttrs "default" (filter (submodule:
+      (submodule.name == null || submodule.name == name) &&
+      (
+        (length submodule.tags == 0) ||
+        (length (intersectLists submodule.tags tags)) > 0
+      )
+    ) config.submodules.defaults);
+
+  submoduleWithSpecialArgs = opts: specialArgs:
+    let
+      opts' = toList opts;
+      inherit (lib.modules) evalModules;
+    in
+    mkOptionType rec {
+      name = "submodule";
+      check = x: isAttrs x || isFunction x;
+      merge = loc: defs:
+        let
+          coerce = def: if isFunction def then def else { config = def; };
+          modules = opts' ++ map (def: { _file = def.file; imports = [(coerce def.value)]; }) defs;
+        in (evalModules {
+          inherit modules specialArgs;
+          prefix = loc;
+        }).config;
+      getSubOptions = prefix: (evalModules
+        { modules = opts'; inherit prefix specialArgs;
+          # This is a work-around due to the fact that some sub-modules,
+          # such as the one included in an attribute set, expects a "args"
+          # attribute to be given to the sub-module. As the option
+          # evaluation does not have any specific attribute name, we
+          # provide a default one for the documentation.
+          #
+          # This is mandatory as some option declaration might use the
+          # "name" attribute given as argument of the submodule and use it
+          # as the default of option declarations.
+          #
+          # Using lookalike unicode single angle quotation marks because
+          # of the docbook transformation the options receive. In all uses
+          # > and < wouldn't be encoded correctly so the encoded values
+          # would be used, and use of `<` and `>` would break the XML document.
+          # It shouldn't cause an issue since this is cosmetic for the manual.
+          args.name = "‹name›";
+        }).options;
+      getSubModules = opts';
+      substSubModules = m: submoduleWithSpecialArgs m specialArgs;
+      functor = (defaultFunctor name) // {
+        # Merging of submodules is done as part of mergeOptionDecls, as we have to annotate
+        # each submodule with its location.
+        payload = [];
+        binOp = lhs: rhs: [];
+      };
+    };
+
+  submoduleDefinitionOptions = {
+    name = mkOption {
+      description = "Module name";
+      type = types.str;
+    };
+
+    description = mkOption {
+      description = "Module description";
+      type = types.str;
+      default = "";
+    };
+
+    version = mkOption {
+      description = "Module version";
+      type = types.str;
+      default = "1.0.0";
+    };
+
+    tags = mkOption {
+      description = "List of submodule tags";
+      type = types.listOf types.str;
+      default = [];
+    };
+  };
+
+  submoduleOptions = {
+    options.submodule = submoduleDefinitionOptions;
+  };
+
+  specialArgs = cfg.specialArgs // {
+    parentConfig = config;
+  };
+
+  findSubmodule = {name, version ? null, latest ? true}: let
+    matchingSubmodules = filter (el:
+      el.definition.name == name &&
+      (if version != null then
+        if hasPrefix "~" version
+        then (builtins.match (removePrefix "~" version) el.definition.version) != null
+        else el.definition.version == version
+      else true)
+    ) cfg.imports;
+
+    versionSortedSubmodules = sort (s1: s2:
+      if builtins.compareVersions s1.definition.version s2.definition.version > 0
+      then true else false
+    ) matchingSubmodules;
+
+    matchingModule =
+      if length versionSortedSubmodules == 0
+      then throw "No module found ${name}/${if version == null then "latest" else version}"
+      else head versionSortedSubmodules;
+  in matchingModule;
+in {
+  options = {
+    submodules.specialArgs = mkOption {
+      description = "Special args to pass to submodules. These arguments can be used for imports";
+      type = types.attrs;
+      default = {};
+    };
+
+    submodules.defaults = mkOption {
+      description = "List of defaults to apply to submodule instances";
+      type = types.listOf (types.submodule ({config, ...}: {
+        options = {
+          name = mkOption {
+            description = "Name of the submodule to apply defaults to";
+            type = types.nullOr types.str;
+            default = null;
+          };
+
+          tags = mkOption {
+            description = "List of tags to apply defaults to";
+            type = types.listOf types.str;
+            default = [];
+          };
+
+          default = mkOption {
+            description = "Default to apply to submodule instance";
+            type = types.unspecified;
+            default = {};
+          };
+        };
+      }));
+      default = [];
+    };
+
+    submodules.propagate = mkOption {
+      description = "Whether to propagate defaults and imports to submodule's submodules";
+      type = types.bool;
+      default = false;
+    };
+
+    submodules.imports = mkOption {
+      description = "List of submodule imports";
+      type = types.listOf (
+        types.coercedTo
+          types.path
+          (module: {inherit module;})
+          (types.submodule ({name, config, ...}: let
+            submoduleDefinition = (evalModules {
+              inherit specialArgs;
+              modules = config.modules ++ [submoduleOptions];
+              check = false;
+            }).config.submodule;
+          in {
+            options = {
+              module = mkOption {
+                description = "Module defining submodule";
+                type = types.unspecified;
+              };
+
+              modules = mkOption {
+                description = "List of modules defining submodule";
+                type = types.listOf types.unspecified;
+                default = [config.module];
+              };
+
+              definition = submoduleDefinitionOptions;
+            };
+
+            config.definition = {
+              inherit (submoduleDefinition) name description version tags;
+            };
+          })
+        )
+      );
+      default = [];
+    };
+
+    submodules.instances = mkOption {
+      description = "Attribute set of submodule instances";
+      type = types.attrsOf (types.submodule ({name, config, ...}: let
+        # submodule associated with
+        submodule = findSubmodule {
+          name = config.submodule;
+          version = config.version;
+        };
+
+        # definition of a submodule
+        submoduleDefinition = submodule.definition;
+
+        # submodule defaults
+        defaults = getDefaults submoduleDefinition.name submoduleDefinition.tags;
+      in {
+        options = {
+          name = mkOption {
+            description = "Submodule instance name";
+            type = types.str;
+            default = name;
+          };
+
+          submodule = mkOption {
+            description = "Name of the submodule to use";
+            type = types.str;
+            default = name;
+          };
+
+          version = mkOption {
+            description = ''
+              Version of submodule to use, if version starts with "~" it is
+              threated as regex pattern for example "~1.0.*"
+            '';
+            type = types.nullOr types.str;
+            default = null;
+          };
+
+          config = mkOption {
+            description = "Submodule instance ${config.name} for ${submoduleDefinition.name}:${submoduleDefinition.version} config";
+            type = submoduleWithSpecialArgs ({...}: {
+              imports = submodule.modules ++ defaults ++ [submoduleOptions];
+              _module.args.name = config.name;
+              _module.args.submodule = config;
+            }) specialArgs;
+            default = {};
+          };
+        };
+      }));
+    };
+    default = {};
+  };
+
+  config = {
+    submodules.specialArgs.kubenix = kubenix;
+    submodules.defaults = [(mkIf cfg.propagate {
+      default = {
+        imports = [./submodules.nix];
+        submodules = {
+          defaults = cfg.defaults;
+          imports = cfg.imports;
+        };
+      };
+    })];
+  };
+}

modules/testing/default.nix

@@ -0,0 +1,258 @@
+{ config, pkgs, lib, kubenix, ... }:
+
+with lib;
+
+let
+  cfg = config.testing;
+  parentConfig = config;
+
+  nixosTesting = import <nixpkgs/nixos/lib/testing.nix> {
+    inherit pkgs;
+    system = "x86_64-linux";
+  };
+
+  kubernetesBaseConfig = { config, pkgs, lib, nodes, ... }: let
+    master = findFirst
+      (node: any (role: role == "master") node.config.services.kubernetes.roles)
+      (throw "no master node")
+      (attrValues nodes);
+    extraHosts = ''
+      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
+      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
+      ${concatMapStringsSep "\n"
+        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
+        (attrValues nodes)}
+    '';
+  in {
+    imports = [ <nixpkgs/nixos/modules/profiles/minimal.nix> ];
+
+    config = mkMerge [{
+      boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+      virtualisation.memorySize = mkDefault 2048;
+      virtualisation.cores = mkDefault 16;
+      virtualisation.diskSize = mkDefault 4096;
+      networking = {
+        inherit extraHosts;
+        domain = "my.xzy";
+        nameservers = ["10.0.0.254"];
+        firewall = {
+          allowedTCPPorts = [
+            10250 # kubelet
+          ];
+          trustedInterfaces = ["docker0" "cni0"];
+
+          extraCommands = concatMapStrings  (node: ''
+            iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+          '') (attrValues nodes);
+        };
+      };
+      environment.systemPackages = [ pkgs.kubectl ];
+      environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
+      services.flannel.iface = "eth1";
+      services.kubernetes = {
+        easyCerts = true;
+        apiserver = {
+          securePort = 443;
+          advertiseAddress = master.config.networking.primaryIPAddress;
+        };
+        masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
+      };
+
+      systemd.extraConfig = "DefaultLimitNOFILE=1048576";
+    }
+    (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
+      networking.firewall.allowedTCPPorts = [
+        443 # kubernetes apiserver
+      ];
+    })];
+  };
+
+  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
+    nixosTesting.makeTest {
+      inherit name;
+
+      nodes.kube = { config, pkgs, lib, nodes, ... }: {
+        imports = [ kubernetesBaseConfig ];
+        services.kubernetes = {
+          roles = ["master" "node"];
+          flannel.enable = false;
+          kubelet = {
+            networkPlugin = "cni";
+            cni.config = [{
+              name = "mynet";
+              type = "bridge";
+              bridge = "cni0";
+              addIf = true;
+              ipMasq = true;
+              isGateway = true;
+              ipam = {
+                type = "host-local";
+                subnet = "10.1.0.0/16";
+                gateway = "10.1.0.1";
+                routes = [{
+                  dst = "0.0.0.0/0";
+                }];
+              };
+            }];
+          };
+        };
+        networking.primaryIPAddress = mkForce "192.168.1.1";
+      };
+
+      testScript = ''
+        startAll;
+
+        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
+
+        ${testScript}
+      '';
+    };
+
+  testOptions = {config, ...}: let
+    modules = [config.module ./test.nix {
+      config._module.args.test = config;
+    }] ++ cfg.defaults;
+
+    test = (kubenix.evalKubernetesModules {
+      check = false;
+      inherit modules;
+    }).config.test;
+
+    evaled =
+      if test.enable
+      then builtins.trace "testing ${test.name}" (kubenix.evalKubernetesModules {
+        inherit modules;
+      })
+      else {success = false;};
+  in {
+    options = {
+      name = mkOption {
+        description = "test name";
+        type = types.str;
+        internal = true;
+      };
+
+      description = mkOption {
+        description = "test description";
+        type = types.str;
+        internal = true;
+      };
+
+      enable = mkOption {
+        description = "Whether to enable test";
+        type = types.bool;
+        internal = true;
+      };
+
+      module = mkOption {
+        description = "Module defining submodule";
+        type = types.unspecified;
+      };
+
+      evaled = mkOption {
+        description = "Wheter test was evaled";
+        type = types.bool;
+        default =
+          if cfg.throwError
+          then if evaled.config.test.assertions != [] then true else true
+          else (builtins.tryEval evaled.config.test.assertions).success;
+        internal = true;
+      };
+
+      success = mkOption {
+        description = "Whether test was success";
+        type = types.bool;
+        internal = true;
+        default = false;
+      };
+
+      assertions = mkOption {
+        description = "Test result";
+        type = types.unspecified;
+        internal = true;
+        default = [];
+      };
+
+      test = mkOption {
+        description = "Test derivation to run";
+        type = types.nullOr types.package;
+        default = null;
+      };
+
+      generated = mkOption {
+        description = "Generated resources";
+        type = types.nullOr types.package;
+        default = null;
+      };
+    };
+
+    config = mkMerge [{
+      inherit (test) name description enable;
+    } (mkIf config.evaled {
+      inherit (evaled.config.test) assertions;
+      success = all (el: el.assertion) config.assertions;
+      test =
+        if cfg.e2e && evaled.config.test.testScript != null
+        then mkKubernetesSingleNodeTest {
+          inherit (evaled.config.test) testScript;
+          name = config.name;
+        } else null;
+      generated = mkIf (hasAttr "kubernetes" evaled.config)
+        (pkgs.writeText "${config.name}-gen.json" (builtins.toJSON evaled.config.kubernetes.generated));
+    })];
+  };
+in {
+  options = {
+    testing.throwError = mkOption {
+      description = "Whether to throw error";
+      type = types.bool;
+      default = true;
+    };
+
+    testing.e2e = mkOption {
+      description = "Whether to enable e2e tests";
+      type = types.bool;
+      default = true;
+    };
+
+    testing.defaults = mkOption {
+      description = "Testing defaults";
+      type = types.coercedTo types.unspecified (value: [value]) (types.listOf types.unspecified);
+      example = literalExample ''{config, ...}: {
+        kubernetes.version = config.kubernetes.version;
+      }'';
+      default = [];
+    };
+
+    testing.tests = mkOption {
+      description = "Attribute set of test cases";
+      default = [];
+      type = types.listOf (types.coercedTo types.path (module: {inherit module;}) (types.submodule testOptions));
+      apply = tests: filter (test: test.enable) tests;
+    };
+
+    testing.testsByName = mkOption {
+      description = "Tests by name";
+      type = types.attrsOf types.attrs;
+      default = listToAttrs (map (test: nameValuePair test.name test) cfg.tests);
+    };
+
+    testing.success = mkOption {
+      description = "Whether testing was a success";
+      type = types.bool;
+      default = all (test: test.success) cfg.tests;
+    };
+
+    testing.result = mkOption {
+      description = "Testing result";
+      type = types.package;
+      default = pkgs.writeText "testing-report.json" (builtins.toJSON {
+        success = cfg.success;
+        tests = map (test: {
+          inherit (test) name description evaled success test;
+          assertions = moduleToAttrs test.assertions;
+        }) (filter (test: test.enable) cfg.tests);
+      });
+    };
+  };
+}

modules/testing/test.nix

@@ -0,0 +1,66 @@
+{ lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.test;
+in {
+  options.test = {
+    name = mkOption {
+      description = "Test name";
+      type = types.str;
+    };
+
+    description = mkOption {
+      description = "Test description";
+      type = types.str;
+    };
+
+    enable = mkOption {
+      description = "Whether to enable test";
+      type = types.bool;
+      default = true;
+    };
+
+    assertions = mkOption {
+      type = types.listOf (types.submodule {
+        options = {
+          assertion = mkOption {
+            description = "assertion value";
+            type = types.bool;
+            default = false;
+          };
+
+          message = mkOption {
+            description = "assertion message";
+            type = types.str;
+          };
+        };
+      });
+      default = [];
+      example = [ { assertion = false; message = "you can't enable this for that reason"; } ];
+      description = ''
+        This option allows modules to express conditions that must
+        hold for the evaluation of the system configuration to
+        succeed, along with associated error messages for the user.
+      '';
+    };
+
+    extraCheckInputs = mkOption {
+      description = "Extra check inputs";
+      type = types.listOf types.package;
+    };
+
+    testScript = mkOption {
+      description = "Script to run as part of testing";
+      type = types.nullOr types.lines;
+      default = null;
+    };
+
+    extraConfig = mkOption {
+      description = "Extra configuration for running test";
+      type = types.unspecified;
+      default = {};
+    };
+  };
+}