diff --git a/default.nix b/default.nix index 6119b2b..d65e507 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,6 @@ -{ pkgs ? import {}, nixosPath ? toString , lib ? pkgs.lib }: +{ pkgs ? import (fetchTarball { + url = "https://github.com/xtruder/nixpkgs/archive/pkgs/k3s/init.tar.gz"; +}) {}, nixosPath ? toString , lib ? pkgs.lib }: with lib; diff --git a/examples/nginx-deployment/test.nix b/examples/nginx-deployment/test.nix index 62ce4a2..c4b80a9 100644 --- a/examples/nginx-deployment/test.nix +++ b/examples/nginx-deployment/test.nix @@ -6,10 +6,11 @@ with lib; imports = [ kubenix.modules.test ./module.nix ]; test = { + distro = "k3s"; name = "nginx-deployment"; description = "Test testing nginx deployment"; testScript = '' - kube.wait_until_succeeds("docker load < ${config.docker.images.nginx.image}") + kube.wait_until_succeeds("docker load --input='${config.docker.images.nginx.image}'") kube.wait_until_succeeds("kubectl apply -f ${config.kubernetes.result}") kube.succeed("kubectl get deployment | grep -i nginx") diff --git a/modules/k3s-airgap-images.nix b/modules/k3s-airgap-images.nix new file mode 100644 index 0000000..80d8b52 --- /dev/null +++ b/modules/k3s-airgap-images.nix @@ -0,0 +1,35 @@ +{ stdenv, fetchurl, k3s }: + +stdenv.mkDerivation rec { + pname = "k3s-airgap-images"; + version = k3s.version; + + src = let + throwError = throw "Unsupported system ${stdenv.hostPlatform.system}"; + in { + x86_64-linux = fetchurl { + url = "https://github.com/rancher/k3s/releases/download/v${version}/k3s-airgap-images-amd64.tar"; + sha256 = "1fiq211vvsnxdzfx9ybb28yyyif08zls7bx3kl8xmv4hrf8xza4i"; + }; + aarch64-linux = fetchurl { + url = "https://github.com/rancher/k3s/releases/download/v${version}/k3s-airgap-images-arm64.tar"; + sha256 = "1xaggiw5h0zndgvdikg7babwd9903n9vabp1dkh53g8al812sfnd"; + }; + armv7l-linux = fetchurl { + url = "https://github.com/rancher/k3s/releases/download/v${version}/k3s-airgap-images-arm.tar"; + sha256 = "1v90wyvj47hz4nphdq7isfbl758yrzg4bx7c73ghmlgvr6p9cdzb"; + }; + }.${stdenv.hostPlatform.system} or throwError; + + preferLocalBuild = true; + dontUnpack = true; + installPhase = "cp $src $out"; + + meta = with stdenv.lib; { + description = "Lightweight Kubernetes. 5 less than k8s. Airgap images."; + homepage = https://k3s.io/; + license = licenses.asl20; + maintainers = [ maintainers.offline ]; + platforms = [ "x86_64-linux" "aarch64-linux" "armv7l-linux" ]; + }; +} diff --git a/modules/test.nix b/modules/test.nix index 449520b..149d698 100644 --- a/modules/test.nix +++ b/modules/test.nix @@ -58,6 +58,12 @@ in { default = null; }; + distro = mkOption { + description = "Kubernetes distro to run the test with. Defaults to 'nixos', other option is 'k3s'"; + type = types.nullOr types.str; + default = null; + }; + extraConfiguration = mkOption { description = "Extra configuration for running test"; type = types.unspecified; diff --git a/modules/testing.nix b/modules/testing.nix index 46b780a..e247f8a 100644 --- a/modules/testing.nix +++ b/modules/testing.nix @@ -4,7 +4,8 @@ with lib; let cfg = config.testing; - + k3s = pkgs.k3s; + k3sAirgapImages = pkgs.callPackage ./k3s-airgap-images.nix {}; toJSONFile = content: builtins.toFile "json" (builtins.toJSON content); nixosTesting = import "${nixosPath}/lib/testing-python.nix" { @@ -111,6 +112,95 @@ let ''; }; + k3sBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let + extraHosts = '' + ${concatMapStringsSep "\n" + (node: let n = node.config.networking; in "${n.primaryIPAddress} ${n.hostName}.${n.domain}") + (attrValues nodes)} + ''; + in { + imports = [ "${toString modulesPath}/profiles/minimal.nix" ]; + + config = mkMerge [{ + virtualisation.memorySize = mkDefault 2048; + virtualisation.cores = mkDefault 16; + virtualisation.diskSize = mkDefault 4096; + virtualisation.docker.enable = true; + networking = { + inherit extraHosts; + domain = "my.xzy"; + nameservers = ["10.43.0.10"]; + firewall = { + trustedInterfaces = ["docker0" "cni0"]; + + extraCommands = concatMapStrings (node: '' + iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT + '') (attrValues nodes); + }; + }; + environment.systemPackages = [ pkgs.kubectl pkgs.docker k3s ]; + environment.variables.KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + systemd.extraConfig = "DefaultLimitNOFILE=1048576"; + systemd.services.seed-docker-images = { + description = "Copy k3s airgap images, and services.kubernetes.kubelet.seedDockerImages"; + wantedBy = ["k3s.service"]; + serviceConfig = { + Type = "oneshot"; + ExecStart = [ + "${pkgs.docker}/bin/docker load --input='${k3sAirgapImages}'" + ] ++ builtins.map + (image : "${pkgs.docker}/bin/docker load --input='${image}'") + config.services.kubernetes.kubelet.seedDockerImages; + }; + }; + systemd.services.k3s = { + description = "Lightweight Kubernetes"; + wantedBy = ["multi-user.target"]; + after = ["network-online.target" "seed-docker-images.service"]; + + serviceConfig = { + # setting --service-cidr 10.0.0.0/24 --cluster-dns 10.0.0.254 + # was flakey, removed and hard coding nameservers to default value + ExecStart = "${k3s}/bin/k3s server --docker"; + Type = "notify"; + KillMode = "process"; + Delegate = "yes"; + LimitNOFILE = "infinity"; + LimitNPROC = "infinity"; + LimitCORE = "infinity"; + TasksMax = "infinity"; + TimeoutStartSec = "0"; + Restart = "always"; + RestartSec = "5s"; + }; + }; + } + (mkIf (any (role: role == "master") config.services.kubernetes.roles) { + networking.firewall.allowedTCPPorts = [ + 443 # kubernetes apiserver + ]; + })]; + }; + + mkK3sTest = { name, testScript, extraConfiguration ? {} }: + + nixosTesting.makeTest { + inherit name; + + nodes.kube = { config, pkgs, nodes, ... }: { + imports = [ k3sBaseConfig extraConfiguration ]; + networking.primaryIPAddress = mkForce "192.168.1.1"; + }; + + testScript = '' + startAll; + + $kube->waitForUnit('k3s.service'); + $kube->waitUntilSucceeds("kubectl get node kube | grep -w Ready"); + + ${testScript} + ''; + }; testOptions = { config, ... }: let modules = [config.module ./test.nix ./base.nix { config = { @@ -209,10 +299,20 @@ let success = all (el: el.assertion) config.assertions; test = if cfg.e2e && evaled.config.test.testScript != null - then mkKubernetesSingleNodeTest { - name = config.name; - inherit (evaled.config.test) testScript extraConfiguration; - } else null; + then + if evaled.config.test.distro == null || + evaled.config.test.distro == "nixos" then + mkKubernetesSingleNodeTest { + name = config.name; + inherit (evaled.config.test) testScript extraConfiguration; + } + else if evaled.config.test.distro == "k3s" then + mkK3sTest { + name = config.name; + inherit (evaled.config.test) testScript extraConfiguration; + } + else throw "invalid test.distro ${evaled.config.test.distro}" + else null; })]; }; in {