technofab/kubenix
1
0
Fork 0
mirror of https://github.com/TECHNOFAB11/kubenix.git synced 2025-12-12 16:10:05 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

ref: reuse upstream k8s testing infra as much as possible

Browse source
This commit is contained in:
David Arnold 2021-05-12 22:34:05 -04:00
parent 0abb1adb9b
commit ffaa63af55
No known key found for this signature in database
GPG key ID: 6D6A936E69C59D08
2 changed files with 69 additions and 129 deletions
Download patch file Download diff file Expand all files Collapse all files

191
modules/testing/runtime/nixos-k8s.nix
View file

@ -1,130 +1,79 @@
# nixos-k8s implements nixos kubernetes testing runtime
{ nixosPath, config, pkgs, lib, kubenix, ... }:
{
nixosPath
, config
, pkgs
, lib
, system ? "x86_64-linux"
, ...
}:
with lib;
let
testConfig = config.evaled.config;
testing = config.testing;
kubeconfig = "/etc/${config.services.kubernetes.pki.etcClusterAdminKubeconfig}";
nixosTesting = import "${nixosPath}/lib/testing.nix" {
inherit pkgs;
system = "x86_64-linux";
};
kubernetesBaseConfig = { modulesPath, config, pkgs, lib, nodes, ... }: let
master = findFirst
(node: any (role: role == "master") node.config.services.kubernetes.roles)
(throw "no master node")
(attrValues nodes);
extraHosts = ''
${master.config.networking.primaryIPAddress} etcd.${config.networking.domain}
${master.config.networking.primaryIPAddress} api.${config.networking.domain}
${concatMapStringsSep "\n"
(node: let n = node.config.networking; in "${n.primaryIPAddress} ${n.hostName}.${n.domain}")
(attrValues nodes)}
'';
in {
imports = [ "${toString modulesPath}/profiles/minimal.nix" ];
config = mkMerge [
# base configuration for master and nodes
{
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
virtualisation.memorySize = mkDefault 2048;
virtualisation.cores = mkDefault 16;
virtualisation.diskSize = mkDefault 4096;
networking = {
inherit extraHosts;
domain = "my.xzy";
nameservers = ["10.0.0.254"];
firewall = {
allowedTCPPorts = [
10250 # kubelet
];
trustedInterfaces = ["docker0" "cni0"];
extraCommands = concatMapStrings (node: ''
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
'') (attrValues nodes);
};
};
environment.systemPackages = [ pkgs.kubectl ];
environment.variables.KUBECONFIG = "/etc/kubernetes/cluster-admin.kubeconfig";
services.flannel.iface = "eth1";
services.kubernetes = {
easyCerts = true;
apiserver = {
securePort = 443;
advertiseAddress = master.config.networking.primaryIPAddress;
};
masterAddress = "${master.config.networking.hostName}.${master.config.networking.domain}";
seedDockerImages = mkIf (elem "docker" testConfig._m.features) testConfig.docker.export;
};
systemd.extraConfig = "DefaultLimitNOFILE=1048576";
systemd.services.copy-certs = {
description = "Share k8s certificates with host";
script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
after = [ "kubernetes.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
};
}
# configuration only applied on master nodes
(mkIf (any (role: role == "master") config.services.kubernetes.roles) {
networking.firewall.allowedTCPPorts = [
443 # kubernetes apiserver
];
})
];
};
mkKubernetesSingleNodeTest = { name, testScript, extraConfiguration ? {} }:
nixosTesting.makeTest {
inherit name;
nodes.kube = { config, pkgs, nodes, ... }: {
imports = [ kubernetesBaseConfig extraConfiguration ];
services.kubernetes = {
roles = ["master" "node"];
flannel.enable = false;
kubelet = {
networkPlugin = "cni";
cni.config = [{
name = "mynet";
type = "bridge";
bridge = "cni0";
addIf = true;
ipMasq = true;
isGateway = true;
ipam = {
type = "host-local";
subnet = "10.1.0.0/16";
gateway = "10.1.0.1";
routes = [{
dst = "0.0.0.0/0";
}];
};
# how we differ from the standard configuration of mkKubernetesBaseTest
extraConfiguration = { config, pkgs, lib, nodes, ...}: {
virtualisation.memorySize = mkDefault 2048;
networking = {
nameservers = ["10.0.0.254"];
firewall = {
trustedInterfaces = ["docker0" "cni0"];
};
};
services.kubernetes = {
seedDockerImages = mkIf (elem "docker" config._m.features) config.docker.export;
flannel.enable = false;
kubelet = {
networkPlugin = "cni";
cni.config = [{
name = "mynet";
type = "bridge";
bridge = "cni0";
addIf = true;
ipMasq = true;
isGateway = true;
ipam = {
type = "host-local";
subnet = "10.1.0.0/16";
gateway = "10.1.0.1";
routes = [{
dst = "0.0.0.0/0";
}];
};
};
networking.primaryIPAddress = mkForce "192.168.1.1";
};
testScript = ''
startAll;
$kube->waitUntilSucceeds("kubectl get node kube.my.xzy | grep -w Ready");
${testScript}
'';
}];
};
systemd.extraConfig = "DefaultLimitNOFILE=1048576";
systemd.services.copy-certs = {
description = "Share k8s certificates with host";
script = "cp -rf /var/lib/kubernetes/secrets /tmp/xchg/";
after = [ "kubernetes.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
};
};
in {
script = ''
machine1.succeed("${testing.testScript} --kube-config=${kubeconfig}")
'';
test =
with import "${nixosPath}/tests/kubernetes/base.nix" { inherit pkgs system; };
mkKubernetesSingleNodeTest {
inherit extraConfiguration;
inherit (config) name;
test = script;
};
in
{
options = {
runtime.nixos-k8s = {
driver = mkOption {
@ -133,9 +82,7 @@ in {
internal = true;
};
};
runtime.nixos-k8s.driver = mkKubernetesSingleNodeTest {
inherit (config) name testScript;
};
};
runtime.nixos-k8s.driver = test.driver;
}