From ffaa63af55feb115b20af8b70b02393ff0ca06ab Mon Sep 17 00:00:00 2001
From: David Arnold <dar@xoe.solutions>
Date: Wed, 12 May 2021 22:34:05 -0400
Subject: [PATCH] ref: reuse upstream k8s testing infra as much as possible

---
 modules/testing/runtime/nixos-k8s.nix         | 191 +++++++-----------
 .../runtime/nixos-k8s/runtime-options.nix     |   7 -
 2 files changed, 69 insertions(+), 129 deletions(-)
 delete mode 100644 modules/testing/runtime/nixos-k8s/runtime-options.nix

diff --git a/modules/testing/runtime/nixos-k8s.nix b/modules/testing/runtime/nixos-k8s.nix
index 013beec..3072265 100644
--- a/modules/testing/runtime/nixos-k8s.nix
+++ b/modules/testing/runtime/nixos-k8s.nix
@@ -1,130 +1,79 @@
 # nixos-k8s implements nixos kubernetes testing runtime
 
-{ nixosPath, config, pkgs, lib, kubenix, ... }:
+{
+  nixosPath
+, config
+, pkgs
+, lib
+, system ? "x86_64-linux"
+, ...
+}:
+
+with lib;
 
 let
-  testConfig = config.evaled.config;
+  testing = config.testing;
+  kubeconfig = "/etc/${config.services.kubernetes.pki.etcClusterAdminKubeconfig}";
 
-  nixosTesting = import "${nixosPath}/lib/testing.nix" {
-    inherit pkgs;
-    system = "x86_64-linux";
-  };
-
-  kubernetesBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
-    master = findFirst
-      (node: any (role: role == "master") node.config.services.kubernetes.roles)
-      (throw "no master node")
-      (attrValues nodes);
-    extraHosts = ''
-      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
-      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
-      ${concatMapStringsSep "\n"
-        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
-        (attrValues nodes)}
-    '';
-  in {
-    imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
-
-    config = mkMerge [
-      # base configuration for master and nodes
-      {
-        boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
-        virtualisation.memorySize = mkDefault 2048;
-        virtualisation.cores = mkDefault 16;
-        virtualisation.diskSize = mkDefault 4096;
-        networking = {
-          inherit extraHosts;
-          domain = "my.xzy";
-          nameservers = ["10.0.0.254"];
-          firewall = {
-            allowedTCPPorts = [
-              10250 # kubelet
-            ];
-            trustedInterfaces = ["docker0" "cni0"];
-
-            extraCommands = concatMapStrings  (node: ''
-              iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
-            '') (attrValues nodes);
-          };
-        };
-        environment.systemPackages = [ pkgs.kubectl ];
-        environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
-        services.flannel.iface = "eth1";
-        services.kubernetes = {
-          easyCerts = true;
-          apiserver = {
-            securePort = 443;
-            advertiseAddress = master.config.networking.primaryIPAddress;
-          };
-          masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
-          seedDockerImages = mkIf (elem "docker" testConfig._m.features) testConfig.docker.export;
-        };
-
-        systemd.extraConfig = "DefaultLimitNOFILE=1048576";
-
-        systemd.services.copy-certs = {
-          description = "Share k8s certificates with host";
-          script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
-          after = [ "kubernetes.target" ];
-          wantedBy = [ "multi-user.target" ];
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-          };
-        };
-      }
-
-      # configuration only applied on master nodes
-      (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
-        networking.firewall.allowedTCPPorts = [
-          443 # kubernetes apiserver
-        ];
-      })
-    ];
-  };
-
-  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
-    nixosTesting.makeTest {
-      inherit name;
-
-      nodes.kube = { config, pkgs, nodes, ... }: {
-        imports = [ kubernetesBaseConfig extraConfiguration ];
-        services.kubernetes = {
-          roles = ["master" "node"];
-          flannel.enable = false;
-          kubelet = {
-            networkPlugin = "cni";
-            cni.config = [{
-              name = "mynet";
-              type = "bridge";
-              bridge = "cni0";
-              addIf = true;
-              ipMasq = true;
-              isGateway = true;
-              ipam = {
-                type = "host-local";
-                subnet = "10.1.0.0/16";
-                gateway = "10.1.0.1";
-                routes = [{
-                  dst = "0.0.0.0/0";
-                }];
-              };
+  # how we differ from the standard configuration of mkKubernetesBaseTest
+  extraConfiguration = { config, pkgs, lib, nodes, ...}: {
+    virtualisation.memorySize = mkDefault 2048;
+    networking = {
+      nameservers = ["10.0.0.254"];
+      firewall = {
+        trustedInterfaces = ["docker0" "cni0"];
+      };
+    };
+    services.kubernetes = {
+      seedDockerImages = mkIf (elem "docker" config._m.features) config.docker.export;
+      flannel.enable = false;
+      kubelet = {
+        networkPlugin = "cni";
+        cni.config = [{
+          name = "mynet";
+          type = "bridge";
+          bridge = "cni0";
+          addIf = true;
+          ipMasq = true;
+          isGateway = true;
+          ipam = {
+            type = "host-local";
+            subnet = "10.1.0.0/16";
+            gateway = "10.1.0.1";
+            routes = [{
+              dst = "0.0.0.0/0";
             }];
           };
-        };
-        networking.primaryIPAddress = mkForce "192.168.1.1";
-      };
-
-      testScript = ''
-        startAll;
-
-        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
-
-        ${testScript}
-      '';
+        }];
     };
+    systemd.extraConfig = "DefaultLimitNOFILE=1048576";
+    systemd.services.copy-certs = {
+      description = "Share k8s certificates with host";
+      script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
+      after = [ "kubernetes.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+    };
+  };
 
-in {
+  script = ''
+    machine1.succeed("${testing.testScript} --kube-config=${kubeconfig}")
+  '';
+
+  test =
+    with import "${nixosPath}/tests/kubernetes/base.nix" { inherit pkgs system; };
+    mkKubernetesSingleNodeTest {
+      inherit extraConfiguration;
+      inherit (config) name;
+      test = script;
+    };
+  
+
+in
+{
   options = {
     runtime.nixos-k8s = {
       driver = mkOption {
@@ -133,9 +82,7 @@ in {
         internal = true;
       };
     };
-
-    runtime.nixos-k8s.driver = mkKubernetesSingleNodeTest {
-      inherit (config) name testScript;
-    };
   };
+
+  runtime.nixos-k8s.driver = test.driver;
 }
diff --git a/modules/testing/runtime/nixos-k8s/runtime-options.nix b/modules/testing/runtime/nixos-k8s/runtime-options.nix
deleted file mode 100644
index 894a1ed..0000000
--- a/modules/testing/runtime/nixos-k8s/runtime-options.nix
+++ /dev/null
@@ -1,7 +0,0 @@
-{ config, ... }:
-
-{
-  options.runtime.nixos-k8s = {
-
-  };
-}