{ pkgs, lib, settings, here ? ./., ...}: with import ./base.nix { inherit pkgs lib settings here; }; with lib; let apiserverAddress = "https://${masterAddress}:4443"; masterAddress = settings.master.address; cfssl-apitoken = let apitoken = pkgs.stdenv.mkDerivation { name = "apitoken"; buildCommand = '' head -c ${toString (32 / 2)} /dev/urandom | \ od -An -t x | tr -d ' ' > $out chmod 400 $out ''; }; in # make ca derivation sha depend on initca cfssl output pkgs.stdenv.mkDerivation { name = "cfssl-apitoken"; src = apitoken; buildCommand = '' cp $src $out ''; }; kube-system-bootstrap = with settings; let worker_nodes = pkgs.writeText "kube-worker-nodes" ( builtins.foldl' (a: x: a + " - ${x.address}\n" ) "" settings.workers); grafana_ldap = pkgs.writeText "grafana-ldap.toml" grafana_ldap_toml; in pkgs.stdenv.mkDerivation { name = "kube-system-bootstrap"; src = ../bootstrap; buildCommand = '' share=$out/share/kube-system-bootstrap mkdir -p $out/bin mkdir -p $share/bin mkdir -p $share/config mkdir -p $share/charts export bash="${pkgs.bash}" export apiserver="${settings.master.name}" export apiserverAddress="${settings.master.address}" export initca="${pki.initca}" export cluster="${clusterName}" export fileserver="${fileserver}" export acme_email="${acme_email}" export grafana_smtp_user="$(echo -n ${grafana_smtp_user} | base64 -w0)" export grafana_smtp_password="$(echo -n ${grafana_smtp_password} | base64 -w0)" export grafana_ldap_toml="$(cat ${grafana_ldap} | base64 -w0)" export workers="$(cat ${worker_nodes})" substituteAll $src/bin/initial-kube-system-bootstrap $share/bin/initial-kube-system-bootstrap chmod 755 $share/bin/initial-kube-system-bootstrap substituteAll $src/copy-kube-system-bootstrap $out/bin/copy-kube-system-bootstrap chmod 755 $out/bin/copy-kube-system-bootstrap cd $src/config for i in *; do substituteAll $i $share/config/$i done cd $src/charts for i in *; do substituteAll $i $share/charts/$i done cp $src/bin/* $share/bin ''; }; install-apitoken = '' #!${pkgs.bash}/bin/bash set -e if [ -d /var/lib/cfssl ]; then cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret chown cfssl /var/lib/cfssl/apitoken.secret chmod 640 /var/lib/cfssl/apitoken.secret else mkdir -p /var/lib/kubernetes/secrets cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret chown root /var/lib/kubernetes/secrets/apitoken.secret chmod 600 /var/lib/kubernetes/secrets/apitoken.secret fi ''; kubeMaster = { services.cfssl.ca = pki.ca.cert; services.cfssl.caKey = pki.ca.key; services.kubernetes = { roles = [ "master" ]; inherit apiserverAddress; masterAddress = settings.master.name; clusterCidr = settings.cidr; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${pki.initca}/ca"; kubelet = { clusterDomain = "${settings.clusterName}.local"; }; apiserver = { advertiseAddress = masterAddress; authorizationMode = [ "Node" "RBAC" ]; allowPrivileged = true; securePort = 4443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}"; extraSANs = settings.master.extraSANs; # verbosity = 4; }; controllerManager = { bindAddress = masterAddress; extraOpts = "--authorization-always-allow-paths=/healthz,/metrics"; }; scheduler.address = masterAddress; addonManager.enable = true; addons = { dns = { enable = true; clusterDomain = "${settings.clusterName}.local"; reconcileMode = "EnsureExists"; }; }; }; services.etcd = { listenClientUrls = [ "https://${masterAddress}:2379" ]; }; networking.firewall = { allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm pkgs.kubectl kube-system-bootstrap ]; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "cfssl.service" ]; before = [ "cfssl.target" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; Restart = "on-failure"; }; }; systemd.services.cfssl-restart = { enable = true; startAt = "00/6:00"; description = "Restrart cfssl which regularly locks up"; script = "systemctl restart cfssl.service"; }; }; kubeWorker = { services.kubernetes = rec { roles = [ "node" ]; inherit apiserverAddress; masterAddress = settings.master.name; clusterCidr = settings.cidr; kubelet.clusterDomain = "${settings.clusterName}.local"; }; networking = { firewall = { enable = true; allowedTCPPorts = [ 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "certmgr.service" ]; before = [ "certmgr.service" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; Restart = "on-failure"; }; }; }; mkApiServer = host: self: { deployment.targetHost = host.address; require = [ (baseNixos host.name) kubeMaster ]; }; mkWorker = host: self: { deployment.targetHost = host.address; require = [ (baseNixos host.name) kubeWorker ]; }; master = { "${settings.master.name}" = mkApiServer settings.master; }; deployment = builtins.foldl' (a: x: a // { "${x.name}" = mkWorker x; }) master settings.workers; in deployment