Update k0 to new-style nix modules
This commit is contained in:
Jonas Juselius
384
modules.bak/k8s.nix
Normal file
384
modules.bak/k8s.nix
Normal file
@@ -0,0 +1,384 @@
|
||||
{ pkgs, lib, config, ...}:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.cluster;
|
||||
|
||||
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
|
||||
|
||||
masterAddress = cfg.k8s.master.address;
|
||||
|
||||
apiserverAddress = "https://${masterAddress}:4443";
|
||||
|
||||
cfssl-apitoken =
|
||||
let
|
||||
apitoken = pkgs.stdenv.mkDerivation {
|
||||
name = "apitoken";
|
||||
buildCommand = ''
|
||||
head -c ${toString (32 / 2)} /dev/urandom | \
|
||||
od -An -t x | tr -d ' ' > $out
|
||||
chmod 400 $out
|
||||
'';
|
||||
};
|
||||
in
|
||||
# make ca derivation sha depend on initca cfssl output
|
||||
pkgs.stdenv.mkDerivation {
|
||||
name = "cfssl-apitoken";
|
||||
src = apitoken;
|
||||
buildCommand = ''
|
||||
cp $src $out
|
||||
'';
|
||||
};
|
||||
|
||||
cluster-scripts =
|
||||
let
|
||||
first = builtins.head cfg.k8s.ingressNodes;
|
||||
rest = builtins.tail cfg.k8s.ingressNodes;
|
||||
ingressNodes = builtins.foldl' (a: x:
|
||||
a + ",${x}") first rest;
|
||||
ingressReplicaCount =
|
||||
builtins.toString (builtins.length cfg.k8s.ingressNodes);
|
||||
show-kubernetes-charts-config = ''
|
||||
#!/usr/bin/env bash
|
||||
cat << EOF
|
||||
# Generated by show-kubernetes-charts-config
|
||||
# $(date)
|
||||
# Charts in ${kubernetes-charts}/share/kubernetes-charts
|
||||
|
||||
vars=(
|
||||
initca="${pki.initca}"
|
||||
apiserver="${cfg.k8s.master.name}"
|
||||
cluster="${cfg.clusterName}"
|
||||
ingress_nodes="${ingressNodes}"
|
||||
ingress_replica_count="${ingressReplicaCount}"
|
||||
fileserver="${cfg.k8s.fileserver}"
|
||||
acme_email="${cfg.k8s.charts.acme_email}"
|
||||
grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)"
|
||||
grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)"
|
||||
)
|
||||
|
||||
# -------------------- no modifications below --------------------
|
||||
EOF
|
||||
cat << '"'"'EOF'"'"'
|
||||
apply=''${KUBECTL_CMD:-apply}
|
||||
|
||||
substitute_all () {
|
||||
local x i k v subs
|
||||
x="$(</dev/stdin)"
|
||||
subs=("$@")
|
||||
for i in "''${subs[@]}"; do
|
||||
k=$(echo "$i" | cut -d= -f1)
|
||||
v=$(echo "$i" | cut -d= -f2 | sed '"'"'s,/,\\/,g'"'"')
|
||||
x=$(echo "$x" | sed "s/@$k@/$v/g")
|
||||
done
|
||||
echo "$x"
|
||||
}
|
||||
|
||||
substitute_defaults () {
|
||||
substitute_all "''${vars[@]}"
|
||||
}
|
||||
|
||||
create_namespace () {
|
||||
kubectl get ns $namespace >/dev/null 2>&1 || kubectl create ns $namespace
|
||||
}
|
||||
|
||||
export_vars () {
|
||||
local i
|
||||
for i in "''${vars[@]}"; do eval "$i"; done
|
||||
}
|
||||
|
||||
kubectl_apply () {
|
||||
local x
|
||||
x="$(</dev/stdin)"
|
||||
create_namespace
|
||||
echo "$x" | \
|
||||
substitute_defaults | \
|
||||
kubectl -n $namespace $apply $@ -f -
|
||||
}
|
||||
|
||||
kubectl_apply_files () {
|
||||
local i charts
|
||||
charts=("$@")
|
||||
for i in "''${charts[@]}"; do
|
||||
cat "$i" | kubectl_apply
|
||||
done
|
||||
}
|
||||
|
||||
helm_apply () {
|
||||
local name chart
|
||||
name="$1"; shift
|
||||
chart="$1"; shift
|
||||
create_namespace $namespace
|
||||
cat values.yaml | \
|
||||
substitute_defaults | \
|
||||
helm template -n $namespace -f- $name $chart $@ | \
|
||||
kubectl -n $namespace $apply -f -
|
||||
}
|
||||
|
||||
[ -z "$namespace" ] && echo "namespace must be set" && exit 1
|
||||
|
||||
if [ "$apply" = delete ]; then
|
||||
echo "WARNING: Really delete $(basename $TOP)? YES/no:"
|
||||
read ans
|
||||
[ "$ans" != YES ] && exit 0
|
||||
fi
|
||||
|
||||
export_vars
|
||||
EOF
|
||||
'';
|
||||
copy-kubernetes-charts = ''
|
||||
#!/usr/bin/env bash
|
||||
if [ \$# = 0 ]; then
|
||||
dest=${cfg.clusterName}-charts
|
||||
else
|
||||
dest=\$1
|
||||
fi
|
||||
[ -e \$dest.bak ] && rm -rf \$dest.bak
|
||||
[ -e \$dest ] && mv \$dest \$dest.bak
|
||||
echo "Copying charts into \$dest"
|
||||
cp -r ${kubernetes-charts}/share/kubernetes-charts \$dest
|
||||
chown -R \$USER \$dest
|
||||
chmod -R u+w \$dest
|
||||
$out/bin/show-kubernetes-charts-config > \$dest/config.sh
|
||||
'';
|
||||
in
|
||||
pkgs.stdenv.mkDerivation {
|
||||
name = "cluster-scripts";
|
||||
src = ../scripts;
|
||||
buildCommand = ''
|
||||
mkdir -p $out/bin
|
||||
cp $src/* $out/bin
|
||||
echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config
|
||||
chmod a+x $out/bin/show-kubernetes-charts-config
|
||||
echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts
|
||||
chmod a+x $out/bin/copy-kubernetes-charts
|
||||
'';
|
||||
};
|
||||
|
||||
kubernetes-charts = pkgs.stdenv.mkDerivation rec {
|
||||
name = "kubernetes-charts";
|
||||
src = ../charts;
|
||||
buildCommand = ''
|
||||
mkdir -p $out/share/${name}
|
||||
cp -r $src/* $out/share/${name}
|
||||
'';
|
||||
};
|
||||
|
||||
install-apitoken = ''
|
||||
#!${pkgs.bash}/bin/bash
|
||||
set -e
|
||||
if [ -d /var/lib/cfssl ]; then
|
||||
cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
|
||||
chown cfssl /var/lib/cfssl/apitoken.secret
|
||||
chmod 640 /var/lib/cfssl/apitoken.secret
|
||||
else
|
||||
mkdir -p /var/lib/kubernetes/secrets
|
||||
cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
|
||||
chown root /var/lib/kubernetes/secrets/apitoken.secret
|
||||
chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
|
||||
fi
|
||||
'';
|
||||
|
||||
kubeMaster = {
|
||||
services.cfssl.ca = pki.ca.cert;
|
||||
services.cfssl.caKey = pki.ca.key;
|
||||
services.kubernetes = {
|
||||
roles = [ "master" ];
|
||||
inherit apiserverAddress;
|
||||
masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
|
||||
clusterCidr = cfg.k8s.cidr;
|
||||
pki.genCfsslCACert = false;
|
||||
pki.genCfsslAPIToken = false;
|
||||
pki.caCertPathPrefix = "${pki.initca}/ca";
|
||||
|
||||
kubelet = {
|
||||
# clusterDomain = "${cfg.clusterName}.local";
|
||||
};
|
||||
|
||||
apiserver = {
|
||||
advertiseAddress = masterAddress;
|
||||
authorizationMode = [ "Node" "RBAC" ];
|
||||
allowPrivileged = true;
|
||||
securePort = 4443;
|
||||
insecurePort = 8080;
|
||||
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
|
||||
extraSANs = cfg.k8s.master.extraSANs;
|
||||
# verbosity = 4;
|
||||
};
|
||||
|
||||
controllerManager = {
|
||||
bindAddress = masterAddress;
|
||||
extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
|
||||
};
|
||||
|
||||
scheduler.address = masterAddress;
|
||||
|
||||
addonManager.enable = true;
|
||||
addons = {
|
||||
dns = {
|
||||
enable = true;
|
||||
# clusterDomain = "${cfg.clusterName}.local";
|
||||
reconcileMode = "EnsureExists";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.etcd = {
|
||||
listenClientUrls = [ "https://${masterAddress}:2379" ];
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
|
||||
allowedUDPPorts = [ 53 4053 ];
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.kubernetes-helm
|
||||
pkgs.kubectl
|
||||
cluster-scripts
|
||||
kubernetes-charts
|
||||
];
|
||||
|
||||
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
||||
description = "Kubernetes certmgr bootstrapper";
|
||||
wantedBy = [ "cfssl.service" ];
|
||||
before = [ "cfssl.target" ];
|
||||
script = install-apitoken;
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
|
||||
# systemd.services.cfssl-restart = {
|
||||
# enable = true;
|
||||
# startAt = "00/6:00";
|
||||
# description = "Restrart cfssl which regularly locks up";
|
||||
# script = "systemctl restart cfssl.service";
|
||||
# };
|
||||
|
||||
systemd.services.kube-socat-https-proxy = {
|
||||
enable = true;
|
||||
wantedBy = [ "kubernetes.target" ];
|
||||
after = [ "kubelet.target" ];
|
||||
description = "Proxy TCP port 443 to ingress NodePort at 30443";
|
||||
script = "${pkgs.socat}/bin/socat TCP-LISTEN:443,fork,reuseaddr TCP:127.0.0.1:30443";
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
kubeNode = {
|
||||
services.kubernetes = rec {
|
||||
roles = [ "node" ];
|
||||
inherit apiserverAddress;
|
||||
# masterAddress = cfg.k8s.master.name;
|
||||
masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
|
||||
clusterCidr = cfg.k8s.cidr;
|
||||
# kubelet.clusterDomain = "${cfg.clusterName}.local";
|
||||
kubelet.hostname = "${cfg.hostName}.${cfg.domain}";
|
||||
proxy.hostname = "${cfg.hostName}.${cfg.domain}";
|
||||
};
|
||||
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [ 4194 10250 ];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
};
|
||||
};
|
||||
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
||||
virtualisation.docker.autoPrune.enable = true;
|
||||
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
||||
description = "Kubernetes certmgr bootstrapper";
|
||||
wantedBy = [ "certmgr.service" ];
|
||||
before = [ "certmgr.service" ];
|
||||
script = install-apitoken;
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
options.cluster.k8s = {
|
||||
enable = mkEnableOption "Enable kubernetes";
|
||||
nodes = mkOption {
|
||||
type = types.listOf types.attrs;
|
||||
default = [];
|
||||
};
|
||||
|
||||
fileserver = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
cidr = mkOption {
|
||||
type = types.str;
|
||||
default = "10.0.0.0/16";
|
||||
};
|
||||
|
||||
ingressNodes = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
master = {
|
||||
enable = mkEnableOption "Enable kubernetes master node";
|
||||
|
||||
address = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
extraSANs = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
hw = mkOption {
|
||||
type = types.path;
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
|
||||
node = {
|
||||
enable = mkEnableOption "Enable kubernetes";
|
||||
};
|
||||
|
||||
charts = {
|
||||
acme_email = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
|
||||
grafana_smtp_user = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
|
||||
grafana_smtp_password = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.k8s.enable (
|
||||
mkMerge [
|
||||
(mkIf cfg.k8s.master.enable kubeMaster)
|
||||
(mkIf cfg.k8s.node.enable kubeNode)
|
||||
]
|
||||
);
|
||||
|
||||
imports = [ ./os.nix ];
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user