365 lines
10 KiB
Nix
365 lines
10 KiB
Nix
{ pkgs, lib, config, ...}:
|
|
with lib;
|
|
let
|
|
cfg = config.cluster;
|
|
|
|
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
|
|
|
|
linkerd-certs = import ./linkerd-certs.nix { inherit pkgs; };
|
|
|
|
masterAddress = cfg.k8s.master.address;
|
|
|
|
apiserverAddress = "https://${masterAddress}:4443";
|
|
|
|
cfssl-apitoken =
|
|
let
|
|
apitoken = pkgs.stdenv.mkDerivation {
|
|
name = "apitoken";
|
|
buildCommand = ''
|
|
head -c ${toString (32 / 2)} /dev/urandom | \
|
|
od -An -t x | tr -d ' ' > $out
|
|
chmod 400 $out
|
|
'';
|
|
};
|
|
in
|
|
# make ca derivation sha depend on initca cfssl output
|
|
pkgs.stdenv.mkDerivation {
|
|
name = "cfssl-apitoken";
|
|
src = apitoken;
|
|
buildCommand = ''
|
|
cp $src $out
|
|
'';
|
|
};
|
|
|
|
cluster-scripts =
|
|
let
|
|
ingressNodes = builtins.foldl' (a: x:
|
|
a + ",${x}") "${cfg.k8s.master.name}"
|
|
cfg.k8s.extraIngressNodes;
|
|
ingressReplicaCount =
|
|
builtins.toString (1 + builtins.length cfg.k8s.extraIngressNodes);
|
|
show-kubernetes-charts-config = ''
|
|
#!/usr/bin/env bash
|
|
cat << EOF
|
|
# Generated by show-kubernetes-charts-config
|
|
# $(date)
|
|
# Charts in ${kubernetes-charts}/share/kubernetes-charts
|
|
|
|
vars=(
|
|
initca="${pki.initca}"
|
|
apiserver="${cfg.k8s.master.name}"
|
|
cluster="${cfg.clusterName}"
|
|
ingress_nodes="${ingressNodes}"
|
|
ingress_replica_count="${ingressReplicaCount}"
|
|
fileserver="${cfg.k8s.fileserver}"
|
|
acme_email="${cfg.k8s.charts.acme_email}"
|
|
grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)"
|
|
grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)"
|
|
linkerd_identity_ca="${linkerd-certs.identity}"
|
|
linkerd_webhook_ca="${linkerd-certs.webhook}"
|
|
)
|
|
|
|
EOF
|
|
cat << '"'"'EOF'"'"'
|
|
apply=''${KUBECTL_CMD:-apply}
|
|
|
|
substitute_all () {
|
|
local x i k v subs
|
|
x="$(</dev/stdin)"
|
|
subs=("$@")
|
|
for i in "''${subs[@]}"; do
|
|
k=$(echo "$i" | cut -d= -f1)
|
|
v=$(echo "$i" | cut -d= -f2 | sed '"'"'s,/,\\/,g'"'"')
|
|
x=$(echo "$x" | sed "s/@$k@/$v/g")
|
|
done
|
|
echo "$x"
|
|
}
|
|
|
|
substitute_defaults () {
|
|
substitute_all "''${vars[@]}"
|
|
}
|
|
|
|
create_namespace () {
|
|
kubectl get ns $namespace >/dev/null 2>&1 || kubectl create ns $namespace
|
|
}
|
|
|
|
export_vars () {
|
|
local i
|
|
for i in "''${vars[@]}"; do eval "$i"; done
|
|
}
|
|
|
|
kubectl_apply () {
|
|
local x
|
|
x="$(</dev/stdin)"
|
|
create_namespace
|
|
echo "$x" | \
|
|
substitute_defaults | \
|
|
kubectl -n $namespace $apply $@ -f -
|
|
}
|
|
|
|
kubectl_apply_files () {
|
|
local i charts
|
|
charts=("$@")
|
|
for i in "''${charts[@]}"; do
|
|
cat "$i" | kubectl_apply
|
|
done
|
|
}
|
|
|
|
helm_apply () {
|
|
local name chart
|
|
name="$1"
|
|
chart="$2"
|
|
create_namespace $namespace
|
|
cat values.yaml | \
|
|
substitute_defaults | \
|
|
helm template -n $namespace -f- $name $chart | \
|
|
kubectl -n $namespace $apply -f -
|
|
}
|
|
|
|
[ -z "$namespace" ] && echo "namespace must be set" && exit 1
|
|
export_vars
|
|
EOF
|
|
'';
|
|
copy-kubernetes-charts = ''
|
|
#!/usr/bin/env bash
|
|
if [ \$# = 0 ]; then
|
|
dest=${cfg.clusterName}-charts
|
|
else
|
|
dest=\$1
|
|
fi
|
|
[ -e $dest.bak ] && rm -rf $dest.bak
|
|
[ -e $dest ] && mv $dest $dest.bak
|
|
echo "Copying charts into \$dest"
|
|
cp -r ${kubernetes-charts}/share/kubernetes-charts \$dest
|
|
chown -R \$USER \$dest
|
|
chmod -R u+w \$dest
|
|
$out/bin/show-kubernetes-charts-config > \$dest/config.sh
|
|
'';
|
|
in
|
|
pkgs.stdenv.mkDerivation {
|
|
name = "cluster-scripts";
|
|
src = ../scripts;
|
|
buildCommand = ''
|
|
mkdir -p $out/bin
|
|
cp $src/* $out/bin
|
|
echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config
|
|
chmod a+x $out/bin/show-kubernetes-charts-config
|
|
echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts
|
|
chmod a+x $out/bin/copy-kubernetes-charts
|
|
'';
|
|
};
|
|
|
|
kubernetes-charts = pkgs.stdenv.mkDerivation rec {
|
|
name = "kubernetes-charts";
|
|
src = ../charts;
|
|
buildCommand = ''
|
|
mkdir -p $out/share/${name}
|
|
cp -r $src/* $out/share/${name}
|
|
'';
|
|
};
|
|
|
|
install-apitoken = ''
|
|
#!${pkgs.bash}/bin/bash
|
|
set -e
|
|
if [ -d /var/lib/cfssl ]; then
|
|
cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
|
|
chown cfssl /var/lib/cfssl/apitoken.secret
|
|
chmod 640 /var/lib/cfssl/apitoken.secret
|
|
else
|
|
mkdir -p /var/lib/kubernetes/secrets
|
|
cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
|
|
chown root /var/lib/kubernetes/secrets/apitoken.secret
|
|
chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
|
|
fi
|
|
'';
|
|
|
|
kubeMaster = {
|
|
services.cfssl.ca = pki.ca.cert;
|
|
services.cfssl.caKey = pki.ca.key;
|
|
services.kubernetes = {
|
|
roles = [ "master" ];
|
|
inherit apiserverAddress;
|
|
masterAddress = cfg.k8s.master.name;
|
|
clusterCidr = cfg.k8s.cidr;
|
|
pki.genCfsslCACert = false;
|
|
pki.genCfsslAPIToken = false;
|
|
pki.caCertPathPrefix = "${pki.initca}/ca";
|
|
|
|
kubelet = {
|
|
# clusterDomain = "${cfg.clusterName}.local";
|
|
};
|
|
|
|
apiserver = {
|
|
advertiseAddress = masterAddress;
|
|
authorizationMode = [ "Node" "RBAC" ];
|
|
allowPrivileged = true;
|
|
securePort = 4443;
|
|
insecurePort = 8080;
|
|
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
|
|
extraSANs = cfg.k8s.master.extraSANs;
|
|
# verbosity = 4;
|
|
};
|
|
|
|
controllerManager = {
|
|
bindAddress = masterAddress;
|
|
extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
|
|
};
|
|
|
|
scheduler.address = masterAddress;
|
|
|
|
addonManager.enable = true;
|
|
addons = {
|
|
dns = {
|
|
enable = true;
|
|
# clusterDomain = "${cfg.clusterName}.local";
|
|
reconcileMode = "EnsureExists";
|
|
};
|
|
};
|
|
};
|
|
|
|
services.etcd = {
|
|
listenClientUrls = [ "https://${masterAddress}:2379" ];
|
|
};
|
|
|
|
networking.firewall = {
|
|
allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
|
|
allowedUDPPorts = [ 53 4053 ];
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.kubernetes-helm
|
|
pkgs.kubectl
|
|
cluster-scripts
|
|
kubernetes-charts
|
|
];
|
|
|
|
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
|
description = "Kubernetes certmgr bootstrapper";
|
|
wantedBy = [ "cfssl.service" ];
|
|
before = [ "cfssl.target" ];
|
|
script = install-apitoken;
|
|
serviceConfig = {
|
|
RestartSec = "10s";
|
|
Restart = "on-failure";
|
|
};
|
|
};
|
|
systemd.services.cfssl-restart = {
|
|
enable = true;
|
|
startAt = "00/6:00";
|
|
description = "Restrart cfssl which regularly locks up";
|
|
script = "systemctl restart cfssl.service";
|
|
};
|
|
};
|
|
|
|
kubeNode = {
|
|
services.kubernetes = rec {
|
|
roles = [ "node" ];
|
|
inherit apiserverAddress;
|
|
masterAddress = cfg.k8s.master.name;
|
|
clusterCidr = cfg.k8s.cidr;
|
|
# kubelet.clusterDomain = "${cfg.clusterName}.local";
|
|
};
|
|
|
|
networking = {
|
|
firewall = {
|
|
enable = true;
|
|
allowedTCPPorts = [ 4194 10250 ];
|
|
allowedUDPPorts = [ 53 ];
|
|
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
|
};
|
|
};
|
|
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
|
virtualisation.docker.autoPrune.enable = true;
|
|
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
|
description = "Kubernetes certmgr bootstrapper";
|
|
wantedBy = [ "certmgr.service" ];
|
|
before = [ "certmgr.service" ];
|
|
script = install-apitoken;
|
|
serviceConfig = {
|
|
RestartSec = "10s";
|
|
Restart = "on-failure";
|
|
};
|
|
};
|
|
};
|
|
|
|
in {
|
|
options.cluster.k8s = {
|
|
enable = mkEnableOption "Enable kubernetes";
|
|
nodes = mkOption {
|
|
type = types.listOf types.attrs;
|
|
default = [];
|
|
};
|
|
|
|
fileserver = mkOption {
|
|
type = types.str;
|
|
default = null;
|
|
};
|
|
|
|
cidr = mkOption {
|
|
type = types.str;
|
|
default = "10.0.0.0/16";
|
|
};
|
|
|
|
extraIngressNodes = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [];
|
|
};
|
|
|
|
master = {
|
|
enable = mkEnableOption "Enable kubernetes master node";
|
|
|
|
address = mkOption {
|
|
type = types.str;
|
|
default = null;
|
|
};
|
|
|
|
name = mkOption {
|
|
type = types.str;
|
|
default = null;
|
|
};
|
|
|
|
extraSANs = mkOption {
|
|
type = types.listOf types.str;
|
|
default = [];
|
|
};
|
|
|
|
hw = mkOption {
|
|
type = types.path;
|
|
default = null;
|
|
};
|
|
};
|
|
|
|
node = {
|
|
enable = mkEnableOption "Enable kubernetes";
|
|
};
|
|
|
|
charts = {
|
|
acme_email = mkOption {
|
|
type = types.str;
|
|
default = "";
|
|
};
|
|
|
|
grafana_smtp_user = mkOption {
|
|
type = types.str;
|
|
default = "";
|
|
};
|
|
|
|
grafana_smtp_password = mkOption {
|
|
type = types.str;
|
|
default = "";
|
|
};
|
|
};
|
|
};
|
|
|
|
config = mkIf cfg.k8s.enable (
|
|
mkMerge [
|
|
(mkIf cfg.k8s.master.enable kubeMaster)
|
|
(mkIf cfg.k8s.node.enable kubeNode)
|
|
]
|
|
);
|
|
|
|
imports = [ ./os.nix ];
|
|
|
|
}
|
|
|