wip k3s support

default.nix

@@ -1,4 +1,6 @@
-{ pkgs ? import <nixpkgs> {}, nixosPath ? toString <nixpkgs/nixos>, lib ? pkgs.lib }:
+{ pkgs ? import (fetchTarball {
+    url = "https://github.com/xtruder/nixpkgs/archive/pkgs/k3s/init.tar.gz";
+}) {}, nixosPath ? toString <nixpkgs/nixos>, lib ? pkgs.lib }:
 
 with lib;
 

examples/nginx-deployment/test.nix

@@ -6,10 +6,11 @@ with lib;
   imports = [ kubenix.modules.test ./module.nix ];
 
   test = {
+    distro = "k3s";
     name = "nginx-deployment";
     description = "Test testing nginx deployment";
     testScript = ''
-      kube.wait_until_succeeds("docker load < ${config.docker.images.nginx.image}")
+      kube.wait_until_succeeds("docker load --input='${config.docker.images.nginx.image}'")
       kube.wait_until_succeeds("kubectl apply -f ${config.kubernetes.result}")
 
       kube.succeed("kubectl get deployment | grep -i nginx")

modules/k3s-airgap-images.nix

@@ -0,0 +1,35 @@
+{ stdenv, fetchurl, k3s }:
+
+stdenv.mkDerivation rec {
+  pname = "k3s-airgap-images";
+  version = k3s.version;
+
+  src = let
+    throwError = throw "Unsupported system ${stdenv.hostPlatform.system}";
+  in {
+    x86_64-linux = fetchurl {
+      url = "https://github.com/rancher/k3s/releases/download/v${version}/k3s-airgap-images-amd64.tar";
+      sha256 = "1fiq211vvsnxdzfx9ybb28yyyif08zls7bx3kl8xmv4hrf8xza4i";
+    };
+    aarch64-linux = fetchurl {
+      url = "https://github.com/rancher/k3s/releases/download/v${version}/k3s-airgap-images-arm64.tar";
+      sha256 = "1xaggiw5h0zndgvdikg7babwd9903n9vabp1dkh53g8al812sfnd";
+    };
+    armv7l-linux = fetchurl {
+      url = "https://github.com/rancher/k3s/releases/download/v${version}/k3s-airgap-images-arm.tar";
+      sha256 = "1v90wyvj47hz4nphdq7isfbl758yrzg4bx7c73ghmlgvr6p9cdzb";
+    };
+  }.${stdenv.hostPlatform.system} or throwError;
+
+  preferLocalBuild = true;
+  dontUnpack = true;
+  installPhase = "cp $src $out";
+
+  meta = with stdenv.lib; {
+    description = "Lightweight Kubernetes. 5 less than k8s. Airgap images.";
+    homepage = https://k3s.io/;
+    license = licenses.asl20;
+    maintainers = [ maintainers.offline ];
+    platforms = [ "x86_64-linux" "aarch64-linux" "armv7l-linux" ];
+  };
+}

modules/test.nix

@@ -58,6 +58,12 @@ in {
       default = null;
     };
 
+    distro = mkOption {
+      description = "Kubernetes distro to run the test with. Defaults to 'nixos', other option is 'k3s'";
+      type = types.nullOr types.str;
+      default = null;
+    };
+
     extraConfiguration = mkOption {
       description = "Extra configuration for running test";
       type = types.unspecified;

modules/testing.nix

@@ -4,7 +4,8 @@ with lib;
 
 let
   cfg = config.testing;
-
+  k3s = pkgs.k3s;
+  k3sAirgapImages = pkgs.callPackage ./k3s-airgap-images.nix {};
   toJSONFile = content: builtins.toFile "json" (builtins.toJSON content);
 
   nixosTesting = import "${nixosPath}/lib/testing-python.nix" {
@@ -111,6 +112,95 @@ let
       '';
     };
 
+  k3sBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
+    extraHosts = ''
+      ${concatMapStringsSep "\n"
+        (node: let n = node.config.networking; in "${n.primaryIPAddress}  ${n.hostName}.${n.domain}")
+        (attrValues nodes)}
+    '';
+  in {
+    imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
+
+    config = mkMerge [{
+      virtualisation.memorySize = mkDefault 2048;
+      virtualisation.cores = mkDefault 16;
+      virtualisation.diskSize = mkDefault 4096;
+      virtualisation.docker.enable = true;
+      networking = {
+        inherit extraHosts;
+        domain = "my.xzy";
+        nameservers = ["10.43.0.10"];
+        firewall = {
+          trustedInterfaces = ["docker0" "cni0"];
+
+          extraCommands = concatMapStrings  (node: ''
+            iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
+          '') (attrValues nodes);
+        };
+      };
+      environment.systemPackages = [ pkgs.kubectl pkgs.docker k3s ];
+      environment.variables.KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
+      systemd.extraConfig = "DefaultLimitNOFILE=1048576";
+      systemd.services.seed-docker-images = {
+        description = "Copy k3s airgap images, and services.kubernetes.kubelet.seedDockerImages";
+        wantedBy = ["k3s.service"];
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = [
+            "${pkgs.docker}/bin/docker load --input='${k3sAirgapImages}'"
+          ] ++ builtins.map 
+            (image : "${pkgs.docker}/bin/docker load --input='${image}'") 
+            config.services.kubernetes.kubelet.seedDockerImages;
+        };
+      };
+      systemd.services.k3s = {
+        description = "Lightweight Kubernetes";
+        wantedBy = ["multi-user.target"];
+        after = ["network-online.target" "seed-docker-images.service"];
+        
+        serviceConfig = {
+          # setting --service-cidr 10.0.0.0/24 --cluster-dns 10.0.0.254
+          # was flakey, removed and hard coding nameservers to default value
+          ExecStart = "${k3s}/bin/k3s server --docker";
+          Type = "notify";
+          KillMode = "process";
+          Delegate = "yes";
+          LimitNOFILE = "infinity";
+          LimitNPROC = "infinity";
+          LimitCORE = "infinity";
+          TasksMax = "infinity";
+          TimeoutStartSec = "0";
+          Restart = "always";
+          RestartSec = "5s";
+        };
+      };
+    }
+    (mkIf (any (role: role == "master") config.services.kubernetes.roles) {
+      networking.firewall.allowedTCPPorts = [
+        443 # kubernetes apiserver
+      ];
+    })];
+  };
+
+  mkK3sTest = { name, testScript, extraConfiguration ? {} }:
+  
+    nixosTesting.makeTest {
+      inherit name;
+      
+      nodes.kube = { config, pkgs, nodes, ... }: {
+        imports = [ k3sBaseConfig extraConfiguration ];
+        networking.primaryIPAddress = mkForce "192.168.1.1";
+      };
+
+      testScript = ''
+        startAll;
+
+        $kube->waitForUnit('k3s.service');
+        $kube->waitUntilSucceeds("kubectl get node kube | grep -w Ready");
+
+        ${testScript}
+      '';
+    };
   testOptions = { config, ... }: let
     modules = [config.module ./test.nix ./base.nix {
       config = {
@@ -209,10 +299,20 @@ let
       success = all (el: el.assertion) config.assertions;
       test =
         if cfg.e2e && evaled.config.test.testScript != null
-        then mkKubernetesSingleNodeTest {
-          name = config.name;
-          inherit (evaled.config.test) testScript extraConfiguration;
-        } else null;
+        then 
+          if evaled.config.test.distro == null || 
+             evaled.config.test.distro == "nixos" then 
+            mkKubernetesSingleNodeTest {
+              name = config.name;
+              inherit (evaled.config.test) testScript extraConfiguration;
+            }
+          else if evaled.config.test.distro == "k3s" then
+            mkK3sTest {
+              name = config.name;
+              inherit (evaled.config.test) testScript extraConfiguration;
+            }
+          else throw "invalid test.distro ${evaled.config.test.distro}"
+        else null;
     })];
   };
 in {