diff --git a/lib.nix b/lib.nix index b93a588..e5e191b 100644 --- a/lib.nix +++ b/lib.nix @@ -27,11 +27,11 @@ in rec { loadYAML = path: importJSON (pkgs.runCommand "yaml-to-json" { } "${pkgs.remarshal}/bin/remarshal -i ${path} -if yaml -of json > $out"); - toYAML = config: builtins.readFile (pkgs.runCommand "to-yaml" { + toYAML = config: pkgs.runCommand "to-yaml" { buildInputs = [pkgs.remarshal]; } '' remarshal -i ${pkgs.writeText "to-json" (builtins.toJSON config)} -if json -of yaml > $out - ''); + ''; toBase64 = value: builtins.readFile diff --git a/testing/default.nix b/testing/default.nix index 88fb5ff..40eda10 100644 --- a/testing/default.nix +++ b/testing/default.nix @@ -5,6 +5,91 @@ with lib; let cfg = config.testing; parentConfig = config; + + nixosTesting = import { + inherit pkgs; + system = "x86_64-linux"; + }; + + mkKubernetesBaseTest = + { name, domain ? "my.zyx", test, machines + , extraConfiguration ? null }: + let + masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines)); + master = machines.${masterName}; + extraHosts = '' + ${master.ip} etcd.${domain} + ${master.ip} api.${domain} + ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)} + ''; + kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } '' + mkdir -p $out/bin + makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig" + ''; + in nixosTesting.makeTest { + inherit name; + + nodes = mapAttrs (machineName: machine: + { config, pkgs, lib, nodes, ... }: + mkMerge [ + { + boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*"; + virtualisation.memorySize = mkDefault 1536; + virtualisation.diskSize = mkDefault 4096; + networking = { + inherit domain extraHosts; + primaryIPAddress = mkForce machine.ip; + + firewall = { + allowedTCPPorts = [ + 10250 # kubelet + ]; + trustedInterfaces = ["docker0"]; + + extraCommands = concatMapStrings (node: '' + iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT + '') (attrValues nodes); + }; + }; + environment.systemPackages = [ kubectl ]; + services.flannel.iface = "eth1"; + services.kubernetes = { + easyCerts = true; + inherit (machine) roles; + apiserver = { + securePort = 443; + advertiseAddress = master.ip; + }; + masterAddress = "${masterName}.${config.networking.domain}"; + }; + } + (optionalAttrs (any (role: role == "master") machine.roles) { + networking.firewall.allowedTCPPorts = [ + 443 # kubernetes apiserver + ]; + }) + (optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; })) + (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; })) + ] + ) machines; + + testScript = '' + startAll; + + ${test} + ''; + }; + + mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({ + machines = { + kube = { + roles = ["master" "node"]; + ip = "192.168.1.1"; + }; + }; + } // attrs // { + name = "kubernetes-${attrs.name}-singlenode"; + }); in { options = { testing.throwError = mkOption { @@ -89,13 +174,27 @@ in { internal = true; default = []; }; + + script = mkOption { + description = "Test script"; + type = types.nullOr types.package; + default = null; + }; }; - config = { + config = mkMerge [{ inherit (test) name description enable; - assertions = mkIf config.evaled evaled.config.test.assertions; - success = mkIf config.evaled (all (el: el.assertion) config.assertions); - }; + } (mkIf config.evaled { + inherit (evaled.config.test) assertions; + success = all (el: el.assertion) config.assertions; + script = if evaled.config.test.check != null then mkKubernetesSingleNodeTest { + name = config.name; + test = '' + $kube->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready"); + ${evaled.config.test.check} + ''; + } else null; + })]; }))); apply = tests: filter (test: test.enable) tests; }; @@ -114,6 +213,7 @@ in { tests = map (test: { inherit (test) name description evaled success; assertions = moduleToAttrs test.assertions; + script = test.script; }) (filter (test: test.enable) cfg.tests); }); }; diff --git a/testing/test.nix b/testing/test.nix index 6235abe..ce03781 100644 --- a/testing/test.nix +++ b/testing/test.nix @@ -1,8 +1,10 @@ -{ lib, ... }: +{ lib, config, pkgs, ... }: with lib; -{ +let + cfg = config.test; +in { options.test = { name = mkOption { description = "Test name"; @@ -43,5 +45,16 @@ with lib; succeed, along with associated error messages for the user. ''; }; + + extraCheckInputs = mkOption { + description = "Extra check inputs"; + type = types.listOf types.package; + }; + + check = mkOption { + description = "Script to run as part testing"; + type = types.nullOr types.lines; + default = null; + }; }; } diff --git a/tests/k8s/crd.nix b/tests/k8s/crd.nix index 1b12f1a..96da923 100644 --- a/tests/k8s/crd.nix +++ b/tests/k8s/crd.nix @@ -1,4 +1,4 @@ -{ config, lib, kubenix, ... }: +{ config, lib, kubenix, pkgs, ... }: with lib; @@ -10,13 +10,18 @@ in { ]; test = { - name = "k8s/crd"; + name = "k8s.crd"; description = "Simple test tesing CRD"; enable = builtins.compareVersions config.kubernetes.version "1.8" >= 0; assertions = [{ message = "should have group set"; assertion = cfg.spec.group == "stable.example.com"; }]; + check = '' + $kube->waitUntilSucceeds("kubectl apply -f ${toYAML config.kubernetes.generated}"); + $kube->succeed("kubectl get crds | grep -i crontabs"); + $kube->succeed("kubectl get crontabs | grep -i crontab"); + ''; }; kubernetes.api.customresourcedefinitions.crontabs = { @@ -33,4 +38,20 @@ in { }; }; }; + + kubernetes.customResources = [{ + group = "stable.example.com"; + version = "v1"; + kind = "CronTab"; + plural = "crontabs"; + description = "CronTabs resources"; + module = { + options.schedule = mkOption { + description = "Crontab schedule script"; + type = types.str; + }; + }; + }]; + + kubernetes.api."stable.example.com"."v1".CronTab.crontab.spec.schedule = "* * * * *"; } diff --git a/tests/k8s/simple.nix b/tests/k8s/simple.nix index 104888a..3297857 100644 --- a/tests/k8s/simple.nix +++ b/tests/k8s/simple.nix @@ -1,4 +1,6 @@ -{ config, test, kubenix, ... }: +{ config, test, kubenix, k8s, ... }: + +with k8s; let cfg = config.kubernetes.api.pods.nginx;