From cfe321c0abb4850ca44dc246e894250fcf871482 Mon Sep 17 00:00:00 2001 From: technofab Date: Sun, 7 Apr 2024 11:27:30 +0000 Subject: [PATCH] feat: add tikv --- flake.nix | 1 + nixlets/tikv/default.nix | 6 ++ nixlets/tikv/service.nix | 22 +++++++ nixlets/tikv/statefulSet.nix | 108 +++++++++++++++++++++++++++++++++++ nixlets/tikv/values.nix | 86 ++++++++++++++++++++++++++++ 5 files changed, 223 insertions(+) create mode 100644 nixlets/tikv/default.nix create mode 100644 nixlets/tikv/service.nix create mode 100644 nixlets/tikv/statefulSet.nix create mode 100644 nixlets/tikv/values.nix diff --git a/flake.nix b/flake.nix index b8ecbf3..75cb9a3 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,7 @@ mosquitto = utils.mkNixlet ./nixlets/mosquitto; attic = utils.mkNixlet ./nixlets/attic; postgres = utils.mkNixlet ./nixlets/postgres; + tikv = utils.mkNixlet ./nixlets/tikv; }; }; perSystem = { diff --git a/nixlets/tikv/default.nix b/nixlets/tikv/default.nix new file mode 100644 index 0000000..aa2c183 --- /dev/null +++ b/nixlets/tikv/default.nix @@ -0,0 +1,6 @@ +{...}: { + imports = [ + ./statefulSet.nix + ./service.nix + ]; +} diff --git a/nixlets/tikv/service.nix b/nixlets/tikv/service.nix new file mode 100644 index 0000000..0944821 --- /dev/null +++ b/nixlets/tikv/service.nix @@ -0,0 +1,22 @@ +{values, ...}: { + kubernetes.resources = { + services."${values.uniqueName}-pd" = { + metadata.annotations."service.alpha.kubernetes.io/tolerate-unready-endpoints" = "true"; + spec = { + selector.app = "${values.uniqueName}-pd"; + ports = [ + { + name = "pd-server"; + port = values.pd.service.client_port; + } + { + name = "peer"; + port = values.pd.service.peer_port; + } + ]; + type = values.pd.service.type; + clusterIP = "None"; + }; + }; + }; +} diff --git a/nixlets/tikv/statefulSet.nix b/nixlets/tikv/statefulSet.nix new file mode 100644 index 0000000..5af129a --- /dev/null +++ b/nixlets/tikv/statefulSet.nix @@ -0,0 +1,108 @@ +{values, ...}: { + kubernetes.resources = { + statefulSets."${values.uniqueName}-pd".spec = { + replicas = values.pd.replicaCount; + selector.matchLabels.name = "${values.uniqueName}-pd"; + serviceName = "${values.uniqueName}-pd"; + updateStrategy.type = "RollingUpdate"; + template = { + metadata.labels.name = "${values.uniqueName}-pd"; + spec = { + containers."pd" = { + image = "${values.pd.image.repository}:${values.pd.image.tag}"; + imagePullPolicy = values.pd.image.pullPolicy; + env = [ + { + name = "INITIAL_CLUSTER_SIZE"; + value = "${builtins.toString values.pd.replicaCount}"; + } + { + name = "SET_NAME"; + value = "${values.uniqueName}-pd"; + } + { + name = "MY_POD_IP"; + valueFrom.fieldRef.fieldPath = "status.podIP"; + } + ]; + ports = { + "pd-server".containerPort = values.pd.service.client_port; + "peer".containerPort = values.pd.service.peer_port; + }; + command = [ + "/bin/sh" + "-ec" + '' + HOSTNAME=$(hostname) + PEERS="" + + for i in $(seq 0 $((''${INITIAL_CLUSTER_SIZE} - 1))); do + PEERS="''${PEERS}''${PEERS:+,}''${SET_NAME}-''${i}=http://''${SET_NAME}-''${i}.''${SET_NAME}:${builtins.toString values.pd.service.peer_port}" + done + + /pd-server --name=''${HOSTNAME} \ + --client-urls=http://0.0.0.0:${builtins.toString values.pd.service.client_port} \ + --peer-urls=http://0.0.0.0:${builtins.toString values.pd.service.peer_port} \ + --advertise-client-urls=http://$(MY_POD_IP):${builtins.toString values.pd.service.client_port} \ + --advertise-peer-urls=http://''${HOSTNAME}.''${SET_NAME}:${builtins.toString values.pd.service.peer_port} \ + --initial-cluster ''${PEERS} + '' + ]; + }; + }; + }; + }; + + statefulSets."${values.uniqueName}".spec = { + replicas = values.tikv.replicaCount; + selector.matchLabels.name = "${values.uniqueName}"; + serviceName = "${values.uniqueName}"; + updateStrategy.type = "RollingUpdate"; + template = { + metadata.labels.name = "${values.uniqueName}"; + spec = { + initContainers."check-pd-port" = { + image = "busybox"; + command = ["sh" "-c" "echo STATUS nc -w 1 ${values.uniqueName}-pd:${builtins.toString values.pd.service.client_port}"]; + }; + containers."tikv" = { + image = "${values.tikv.image.repository}:${values.tikv.image.tag}"; + imagePullPolicy = values.tikv.image.pullPolicy; + env = [ + { + name = "MY_POD_IP"; + valueFrom.fieldRef.fieldPath = "status.podIP"; + } + ]; + ports."client".containerPort = values.tikv.service.client_port; + command = [ + "/bin/sh" + "-ecx" + '' + /tikv-server \ + --addr="0.0.0.0:${builtins.toString values.tikv.service.client_port}" \ + --advertise-addr="$(MY_POD_IP):${builtins.toString values.tikv.service.client_port}" \ + --data-dir="/data/tikv" \ + --pd="${values.uniqueName}-pd:${builtins.toString values.pd.service.client_port}" + '' + ]; + volumeMounts."data" = { + name = "${values.uniqueName}-data"; + mountPath = "/data"; + }; + # TODO: liveness and readiness probes + }; + }; + }; + volumeClaimTemplates = [ + { + metadata.name = "${values.uniqueName}-data"; + spec = { + accessModes = ["ReadWriteOnce"]; + resources.requests.storage = values.tikv.storage; + }; + } + ]; + }; + }; +} diff --git a/nixlets/tikv/values.nix b/nixlets/tikv/values.nix new file mode 100644 index 0000000..2b6f5ad --- /dev/null +++ b/nixlets/tikv/values.nix @@ -0,0 +1,86 @@ +{ + lib, + utils, + project, + ... +}: +with lib; { + # for some basic values see https://github.com/helm/examples/blob/4888ba8fb8180dd0c36d1e84c1fcafc6efd81532/charts/hello-world/values.yaml + options = { + pd = utils.mkNestedOption { + replicaCount = mkOption { + type = types.int; + default = 3; + }; + image = utils.mkNestedOption { + repository = mkOption { + type = types.str; + default = "pingcap/pd"; + }; + tag = mkOption { + type = types.str; + default = "latest"; + }; + pullPolicy = mkOption { + type = types.str; + default = "IfNotPresent"; + }; + }; + service = utils.mkNestedOption { + peer_port = mkOption { + type = types.int; + default = 2380; + }; + client_port = mkOption { + type = types.int; + default = 2379; + }; + type = mkOption { + type = types.str; + default = "ClusterIP"; + }; + }; + }; + tikv = utils.mkNestedOption { + replicaCount = mkOption { + type = types.int; + default = 3; + }; + image = utils.mkNestedOption { + repository = mkOption { + type = types.str; + default = "pingcap/tikv"; + }; + tag = mkOption { + type = types.str; + default = "latest"; + }; + pullPolicy = mkOption { + type = types.str; + default = "IfNotPresent"; + }; + }; + service = utils.mkNestedOption { + client_port = mkOption { + type = types.int; + default = 20160; + }; + type = mkOption { + type = types.str; + default = "ClusterIP"; + }; + }; + storage = mkOption { + type = types.str; + default = "5G"; + }; + }; + + # internal + uniqueName = mkOption { + internal = true; + type = types.str; + default = "${project}-tikv"; + }; + }; +}