ref: reuse upstream k8s testing infra as much as possible

modules/testing/runtime/nixos-k8s.nix

@@ -1,96 +1,31 @@
 # nixos-k8s implements nixos kubernetes testing runtime
 
-{ nixosPath, config, pkgs, lib, kubenix, ... }:
+{
+  nixosPath
+, config
+, pkgs
+, lib
+, system ? "x86_64-linux"
+, ...
+}:
+
+with lib;
 
 let
-  testConfig = config.evaled.config;
+  testing = config.testing;
+  kubeconfig = "/etc/${config.services.kubernetes.pki.etcClusterAdminKubeconfig}";
 
-  nixosTesting = import "${nixosPath}/lib/testing.nix" {
+  # how we differ from the standard configuration of mkKubernetesBaseTest
-    inherit pkgs;
+  extraConfiguration = { config, pkgs, lib, nodes, ...}: {
-    system = "x86_64-linux";
-  };
-
-  kubernetesBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
-    master = findFirst
-      (node: any (role: role == "master") node.config.services.kubernetes.roles)
-      (throw "no master node")
-      (attrValues nodes);
-    extraHosts = ''
-      ${master.config.networking.primaryIPAddress}  etcd.${config.networking.domain}
-      ${master.config.networking.primaryIPAddress}  api.${config.networking.domain}
-      ${concatMapStringsSep "\n"
-        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
-        (attrValues nodes)}
-    '';
-  in {
-    imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
-
-    config = mkMerge [
-      # base configuration for master and nodes
-      {
-        boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
     virtualisation.memorySize = mkDefault 2048;
-        virtualisation.cores = mkDefault 16;
-        virtualisation.diskSize = mkDefault 4096;
     networking = {
-          inherit extraHosts;
-          domain = "my.xzy";
       nameservers = ["10.0.0.254"];
       firewall = {
-            allowedTCPPorts = [
-              10250 # kubelet
-            ];
         trustedInterfaces = ["docker0" "cni0"];
-
-            extraCommands = concatMapStrings  (node: ''
-              iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
-            '') (attrValues nodes);
       };
     };
-        environment.systemPackages = [ pkgs.kubectl ];
-        environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
-        services.flannel.iface = "eth1";
     services.kubernetes = {
-          easyCerts = true;
+      seedDockerImages = mkIf (elem "docker" config._m.features) config.docker.export;
-          apiserver = {
-            securePort = 443;
-            advertiseAddress = master.config.networking.primaryIPAddress;
-          };
-          masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
-          seedDockerImages = mkIf (elem "docker" testConfig._m.features) testConfig.docker.export;
-        };
-
-        systemd.extraConfig = "DefaultLimitNOFILE=1048576";
-
-        systemd.services.copy-certs = {
-          description = "Share k8s certificates with host";
-          script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
-          after = [ "kubernetes.target" ];
-          wantedBy = [ "multi-user.target" ];
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-          };
-        };
-      }
-
-      # configuration only applied on master nodes
-      (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
-        networking.firewall.allowedTCPPorts = [
-          443 # kubernetes apiserver
-        ];
-      })
-    ];
-  };
-
-  mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
-    nixosTesting.makeTest {
-      inherit name;
-
-      nodes.kube = { config, pkgs, nodes, ... }: {
-        imports = [ kubernetesBaseConfig extraConfiguration ];
-        services.kubernetes = {
-          roles = ["master" "node"];
       flannel.enable = false;
       kubelet = {
         networkPlugin = "cni";
@@ -111,20 +46,34 @@ let
           };
         }];
     };
+    systemd.extraConfig = "DefaultLimitNOFILE=1048576";
+    systemd.services.copy-certs = {
+      description = "Share k8s certificates with host";
+      script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
+      after = [ "kubernetes.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
     };
-        networking.primaryIPAddress = mkForce "192.168.1.1";
   };
 
-      testScript = ''
+  script = ''
-        startAll;
+    machine1.succeed("${testing.testScript} --kube-config=${kubeconfig}")
-
-        $kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
-
-        ${testScript}
   '';
+
+  test =
+    with import "${nixosPath}/tests/kubernetes/base.nix" { inherit pkgs system; };
+    mkKubernetesSingleNodeTest {
+      inherit extraConfiguration;
+      inherit (config) name;
+      test = script;
     };
   
-in {
+
+in
+{
   options = {
     runtime.nixos-k8s = {
       driver = mkOption {
@@ -133,9 +82,7 @@ in {
         internal = true;
       };
     };
+  };
 
-    runtime.nixos-k8s.driver = mkKubernetesSingleNodeTest {
+  runtime.nixos-k8s.driver = test.driver;
-      inherit (config) name testScript;
-    };
-  };
 }

modules/testing/runtime/nixos-k8s/runtime-options.nix

@@ -1,7 +0,0 @@
-{ config, ... }:
-
-{
-  options.runtime.nixos-k8s = {
-
-  };
-}