{ pkgs, lib, config, ...}: with lib; let cfg = config.cluster; pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; apiserverAddress = "https://${masterAddress}:4443"; masterAddress = cfg.k8s.master.address; cfssl-apitoken = let apitoken = pkgs.stdenv.mkDerivation { name = "apitoken"; buildCommand = '' head -c ${toString (32 / 2)} /dev/urandom | \ od -An -t x | tr -d ' ' > $out chmod 400 $out ''; }; in # make ca derivation sha depend on initca cfssl output pkgs.stdenv.mkDerivation { name = "cfssl-apitoken"; src = apitoken; buildCommand = '' cp $src $out ''; }; kube-system-bootstrap = let workerNodes = pkgs.writeText "kube-worker-nodes" ( builtins.foldl' (a: x: a + " - ${x.address}\n" ) "" cfg.k8s.nodes); grafanaLdap = pkgs.writeText "grafana-ldap.toml" cfg.k8s.bootstrap.grafana_ldap_toml; in pkgs.stdenv.mkDerivation { name = "kube-system-bootstrap"; src = ../bootstrap; buildCommand = '' share=$out/share/kube-system-bootstrap mkdir -p $out/bin mkdir -p $share/bin mkdir -p $share/config mkdir -p $share/charts export bash="${pkgs.bash}" export apiserver="${cfg.k8s.master.name}" export apiserverAddress="${cfg.k8s.master.address}" export initca="${pki.initca}" export cluster="${cfg.clusterName}" export fileserver="${cfg.k8s.fileserver}" export acme_email="${cfg.k8s.bootrstrap.acme_email}" export grafana_smtp_user="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_user} | base64 -w0)" export grafana_smtp_password="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_password} | base64 -w0)" export grafana_ldap_toml="$(cat ${grafanaLdap} | base64 -w0)" export workers="$(cat ${workerNodes})" substituteAll $src/bin/initial-kube-system-bootstrap $share/bin/initial-kube-system-bootstrap chmod 755 $share/bin/initial-kube-system-bootstrap substituteAll $src/copy-kube-system-bootstrap $out/bin/copy-kube-system-bootstrap chmod 755 $out/bin/copy-kube-system-bootstrap cd $src/config for i in *; do substituteAll $i $share/config/$i done cd $src/charts for i in *; do substituteAll $i $share/charts/$i done cp $src/bin/* $share/bin ''; }; install-apitoken = '' #!${pkgs.bash}/bin/bash set -e if [ -d /var/lib/cfssl ]; then cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret chown cfssl /var/lib/cfssl/apitoken.secret chmod 640 /var/lib/cfssl/apitoken.secret else mkdir -p /var/lib/kubernetes/secrets cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret chown root /var/lib/kubernetes/secrets/apitoken.secret chmod 600 /var/lib/kubernetes/secrets/apitoken.secret fi ''; kubeMaster = { services.cfssl.ca = pki.ca.cert; services.cfssl.caKey = pki.ca.key; services.kubernetes = { roles = [ "master" ]; inherit apiserverAddress; masterAddress = cfg.k8s.master.name; clusterCidr = cfg.k8s.cidr; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${pki.initca}/ca"; kubelet = { clusterDomain = "${cfg.clusterName}.local"; }; apiserver = { advertiseAddress = masterAddress; authorizationMode = [ "Node" "RBAC" ]; allowPrivileged = true; securePort = 4443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}"; extraSANs = cfg.k8s.master.extraSANs; # verbosity = 4; }; controllerManager = { bindAddress = masterAddress; extraOpts = "--authorization-always-allow-paths=/healthz,/metrics"; }; scheduler.address = masterAddress; addonManager.enable = true; addons = { dns = { enable = true; clusterDomain = "${cfg.clusterName}.local"; reconcileMode = "EnsureExists"; }; }; }; services.etcd = { listenClientUrls = [ "https://${masterAddress}:2379" ]; }; networking.firewall = { allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm pkgs.kubectl kube-system-bootstrap ]; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "cfssl.service" ]; before = [ "cfssl.target" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; Restart = "on-failure"; }; }; systemd.services.cfssl-restart = { enable = true; startAt = "00/6:00"; description = "Restrart cfssl which regularly locks up"; script = "systemctl restart cfssl.service"; }; }; kubeNode = { services.kubernetes = rec { roles = [ "node" ]; inherit apiserverAddress; masterAddress = cfg.k8s.master.name; clusterCidr = cfg.k8s.cidr; kubelet.clusterDomain = "${cfg.clusterName}.local"; }; networking = { firewall = { enable = true; allowedTCPPorts = [ 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "certmgr.service" ]; before = [ "certmgr.service" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; Restart = "on-failure"; }; }; }; in { options.cluster.k8s = { nodes = mkOption { type = types.attrs; default = {}; }; fileserver = mkOption { type = types.str; default = "127.0.0.1"; }; cidr = mkOption { type = types.str; default = "10.11.0.0/16"; }; master = { enable = mkEnableOption "Enable kubernetes master node"; address = mkOption { type = types.str; default = "127.0.0.1"; }; name = mkOption { type = types.str; default = "apiserver"; }; }; node = { enable = mkEnableOption "Enable kubernetes"; }; bootstrap = { acme_email = mkOption { type = types.str; default = ""; }; grafana_smtp_user = mkOption { type = types.str; default = ""; }; grafana_smtp_password = mkOption { type = types.str; default = ""; }; grafana_ldap = mkOption { type = types.str; default = ""; }; }; }; config = mkMerge [ (mkIf cfg.k8s.master.enable kubeMaster) (mkIf cfg.k8s.node.enable kubeNode) ]; imports = [ ./os.nix ]; }