platform/lib/k8s.nix

{ pkgs, lib, settings, here ? "", ...}:
with lib;
let
  apiserverAddress = "https://${masterAddress}:4443";
  masterAddress = settings.master.address;
  initca = settings.initca;

  cluster-ca = import ./initca.nix { inherit pgks initca; };

  cfssl-apitoken = pkgs.stdenv.mkDerivation {
    name = "cfssl-apitoken";
    buildCommand = ''
      head -c ${toString (32 / 2)} /dev/urandom | \
      od -An -t x | tr -d ' ' > $out
      chmod 400 $out
    '';
  };

  kube-system-bootstrap =
    with settings;
    let
      worker_nodes = pkgs.writeText "worker-nodes.txt" (
        builtins.foldl' (a: x:
          a + "  - ${x.address}\n"
        ) "" settings.workers);
      grafana_ldap = pkgs.writeText "grafana-ldap.toml" grafana_ldap_toml;
    in
    pkgs.stdenv.mkDerivation {
      name = "bootstrap-kube-system";
      src = ../bootstrap;
      buildCommand = ''
        mkdir -p $out/bin
        mkdir -p $out/share/kube-system-bootstrap/config
        mkdir -p $out/share/kube-system-bootstrap/charts

        export bash="${pkgs.bash}"
        export apiserver="${settings.master.address}"
        export initca="${initca}"
        export cluster="${clusterName}"
        export fileserver="${fileserver}"
        export acme_email="${acme_email}"
        export grafana_smtp_user="$(echo -n ${grafana_smtp_user} | base64 -w0)"
        export grafana_smtp_password="$(echo -n ${grafana_smtp_password} | base64 -w0)"
        export grafana_ldap_toml="$(cat ${grafana_ldap} | base64 -w0)"
        export workers="$(cat ${worker_nodes})"

        substituteAll $src/kube-system-bootstrap $out/bin/bootstrap-kube-system
        chmod 755 $out/bin/bootstrap-kube-system

        cd $src/config
        for i in *; do
          substituteAll $i $out/share/kube-system-bootstrap/config/$i
        done

        cd $src/charts
        for i in *; do
          substituteAll $i $out/share/kube-system-bootstrap/charts/$i
        done

        cp $src/bin/* $out/bin
      '';
    };

  install-apitoken = ''
      #!${pkgs.bash}/bin/bash
      set -e
      if [ -d /var/lib/cfssl ]; then
        cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
        chown cfssl /var/lib/cfssl/apitoken.secret
        chmod 640 /var/lib/cfssl/apitoken.secret
      else
        mkdir -p /var/lib/kubernetes/secrets
        cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
        chown root /var/lib/kubernetes/secrets/apitoken.secret
        chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
      fi
    '';

  kubeMaster = {
    services.cfssl.ca = "${cluster-ca}/ca.pem";
    services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
    services.kubernetes = {
      roles = [ "master" ];
      inherit apiserverAddress masterAddress;
      clusterCidr = settings.cidr;
      pki.genCfsslCACert = false;
      pki.genCfsslAPIToken = false;
      pki.caCertPathPrefix = "${cluster-ca}/ca";

      kubelet = {
        clusterDomain = "${settings.clusterName}.local";
      };

      apiserver = {
        advertiseAddress = masterAddress;
        authorizationMode = [ "Node" "RBAC" ];
        allowPrivileged = true;
        securePort = 4443;
        insecurePort = 8080;
        extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
      };

      controllerManager = {
        bindAddress = masterAddress;
        extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
      };

      scheduler.address = masterAddress;

      addonManager.enable = true;
      addons = {
        dns = {
          enable = true;
          clusterDomain = "${settings.clusterName}.local";
          reconcileMode = "EnsureExists";
        };
      };
    };

    services.etcd = {
      listenClientUrls = [ "https://${masterAddress}:2379" ];
    };

    networking.firewall = {
      allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
      allowedUDPPorts = [ 53 4053 ];
    };

    environment.systemPackages = [
      pkgs.kubernetes-helm
      pkgs.kubectl
      kube-system-bootstrap
    ];

    systemd.services.kube-certmgr-apitoken-bootstrap = {
      description = "Kubernetes certmgr bootstrapper";
      wantedBy = [ "cfssl.service" ];
      before = [ "cfssl.target" ];
      script = install-apitoken;
      serviceConfig = {
        RestartSec = "10s";
        Restart = "on-failure";
      };
    };
  };

  kubeWorker = {
    services.kubernetes = rec {
      roles = [ "node" ];
      inherit apiserverAddress masterAddress;
      clusterCidr = settings.cidr;
      kubelet.clusterDomain = "${settings.clusterName}.local";
    };

    networking = {
      firewall = {
        enable = true;
        allowedTCPPorts = [ 4194 10250 ];
        allowedUDPPorts = [ 53 ];
        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
      };
    };
    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
    virtualisation.docker.autoPrune.enable = true;
    systemd.services.kube-certmgr-apitoken-bootstrap = {
      description = "Kubernetes certmgr bootstrapper";
      wantedBy = [ "certmgr.service" ];
      before = [ "certmgr.service" ];
      script = install-apitoken;
      serviceConfig = {
        RestartSec = "10s";
        Restart = "on-failure";
      };
    };
  };

  baseNixos = name: {
    users.extraUsers.admin.openssh.authorizedKeys.keys =
     settings.adminAuthorizedKeys;

    boot.kernel.sysctl = {
      "kernel.mm.transparent_hugepage.enabled" = "never";
      "net.core.somaxconn" = "512";
    };

    imports = [
      ./nixos/configuration.nix
      (here + "/${name}.nix")
    ];
    security.pki.certificateFiles = [
      "${cluster-ca}/ca.pem"
    ];
    # services.glusterfs = {
    #   enable = true;
    #   # tlsSettings = {
    #     # caCert = certs.ca.caFile;
    #     # tlsKeyPath = certs.self.keyFile;
    #     # tlsPem = certs.self.certFile;
    #   };
    # };
    networking = {
      hostName = name;
      extraHosts = settings.clusterHosts;
      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
      firewall.allowedTCPPorts = [ 80 443 111 ];
      firewall.allowedUDPPorts = [ 111 24007 24008 ];
    };
    environment.systemPackages = with pkgs; [
      nfs-utils
    ];
  };

  mkApiServer = host: self:
    {
      deployment.targetHost = host.address;
      require = [
        (baseNixos host.name)
        kubeMaster
      ];
    };

  mkWorker = host: self:
    {
      deployment.targetHost = host.address;
      require = [
        (baseNixos host.name)
        kubeWorker
      ];
    };

  mkHost = host: self:
    {
      deployment.targetHost = host.address;
      require = [
        (baseNixos host.name)
      ];
    };

  master = { "${settings.master.name}" = mkApiServer settings.master; };
in
  builtins.foldl'
    (a: x: a // { "${x.name}" = mkWorker x; }) master settings.workers