platform/lib/k8s.nix

here: { pkgs, lib, settings, ...}:
with lib;
let
  cluster-ca = pkgs.stdenv.mkDerivation {
    name = "cluster-ca";
    src = here + /ca;
    buildCommand = ''
     mkdir -p $out
     cp $src/* $out
    '';
  };

  cfssl-apitoken = pkgs.stdenv.mkDerivation {
    name = "cfssl-apitoken";
    buildCommand = ''
      head -c ${toString (32 / 2)} /dev/urandom | \
      od -An -t x | tr -d ' ' > $out
      chmod 400 $out
    '';
  };

  nixos-kubernetes-join-nodes = workers:
    let
      wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
    in
      pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
        #!/bin/sh
        set -e
        token=$(cat /var/lib/cfssl/apitoken.secret)
        for i in ${wrk}; do
           ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
        done
      '';

  kube-system-bootstrap = pkgs.stdenv.mkDerivation {
    name = "kube-system-bootstrap";
    src = ../kube-system-bootstrap;
    buildCommand = ''
     mkdir -p $out/bin
     mkdir -p $out/share/kube-system-bootstrap
     cp -r $src/* $out/share/kube-system-bootstrap/
     cd $out/bin
     ln -s ../share/kube-system-bootstrap/bin/* .
     ln -s ../share/kube-system-bootstrap/kube-system-bootstrap .
    '';
  };

  install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" ''
    #!${pkgs.bash}/bin/bash
    set -e
    if [ -d /var/lib/cfssl ]; then
      cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
      chown cfssl /var/lib/cfssl/apitoken.secret
      chmod 600 /var/lib/cfssl/apitoken.secret
    fi
    cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
    chown root /var/lib/kubernetes/secrets/apitoken.secret
    chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
  '';

  cidr = "10.10.0.0/16";
in
rec {
  kubeMaster = {
    services.cfssl.ca = "${cluster-ca}/ca.pem";
    services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
    services.kubernetes = {
      roles = [ "master" ];
      masterAddress = settings.master;
      apiserverAddress = settings.apiserverAddress;
      clusterCidr = cidr;
      kubelet.unschedulable = false;
      pki.genCfsslCACert = false;
      pki.genCfsslAPIToken = false;
      pki.caCertPathPrefix = "${cluster-ca}/ca";
      apiserver = {
        advertiseAddress = settings.masterAddress;
        authorizationMode = [ "Node" "RBAC" ];
        securePort = 8443;
        insecurePort = 8080;
        extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
      };
      addons = {
        dns = {
          enable = true;
          # clusterDomain = "local";
          reconcileMode = "EnsureExists";
        };
      };
    };
    networking.firewall = {
      allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
      allowedUDPPorts = [ 53 4053 ];
    };
    environment.systemPackages = [
      pkgs.kubernetes-helm
      (nixos-kubernetes-join-nodes settings.workers)
      kube-system-bootstrap
    ];
  };

  kubeWorker = {
    services.kubernetes = rec {
      roles = [ "node" ];
      clusterCidr = cidr;
      masterAddress = settings.master;
      apiserverAddress = settings.apiserverAddress;
    };
    networking = {
      firewall = {
        enable = true;
        allowedTCPPorts = [ 4194 10250 ];
        allowedUDPPorts = [ 53 ];
        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
      };
    };
    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
    virtualisation.docker.autoPrune.enable = true;
  };

  baseNixos = name: {
    imports = [
      ../nixos/configuration.nix
      (here + "/${name}.nix")
    ];
    security.pki.certificateFiles = [
      "${cluster-ca}/ca.pem"
    ];
    # services.glusterfs = {
    #   enable = true;
    #   # tlsSettings = {
    #     # caCert = certs.ca.caFile;
    #     # tlsKeyPath = certs.self.keyFile;
    #     # tlsPem = certs.self.certFile;
    #   };
    # };
    networking = {
      hostName = name;
      extraHosts = settings.clusterHosts;
      # nameservers = [ masterAddress ];
      # dhcpcd.extraConfig = ''
      #  static domain_name_servers=${masterAddress}
      # '';
      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
      firewall.allowedTCPPorts = [ 80 443 111 ];
      firewall.allowedUDPPorts = [ 111 24007 24008 ];
    };
    users.extraUsers.admin.openssh.authorizedKeys.keys =
     settings.adminAuthorizedKeys;

    systemd.services.kube-certmgr-apitoken-bootstrap = {
      description = "Kubernetes certmgr bootstrapper";
      wantedBy = [ "cfssl.service" ];
      before = [ "cfssl.target" ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        ExecStart = "${install-apitoken}";
      };
    };
  };

  apiserver = ip: name: self:
    {
      deployment.targetHost = ip;
      require = [
        (baseNixos name)
        kubeMaster
      ];
    };

  worker = ip: name: self:
    {
      deployment.targetHost = ip;
      require = [
        (baseNixos name)
        kubeWorker
      ];
    };

  host = ip: name: self:
    {
      deployment.targetHost = ip;
      require = [
        (baseNixos name)
      ];
    };
}