platform/lib/k8s.nix

{ pkgs, masterNode, etcdNodes, clusterHosts, certs, ...}:
let
  kubeApiserver = "https://${masterNode}:8443";
  localApiserver = "http://127.0.0.1:8080";
  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
in
rec {
  etcdConfig = name: {
    services.etcd = {
      inherit name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      keyFile  = certs.etcd.key;
      certFile = certs.etcd.cert;
      trustedCaFile = certs.ca.cert;
      advertiseClientUrls = [ "https://${name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
      ETCDCTL_KEY_FILE  = "${certs.admin.key}";
      ETCDCTL_CERT_FILE = "${certs.admin.cert}";
      ETCDCTL_CA_FILE   = "${certs.ca.cert}";
      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
    };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

   clientConf = instance: {
    server = kubeApiserver;
    keyFile = certs.${instance}.key;
    certFile = certs.${instance}.cert;
    caFile = certs.ca.cert;
  };

  kubeNode = instance: {
    services.kubernetes = rec {
      roles = [ "node" ];
      kubeconfig = clientConf instance;
      kubelet = {
        enable = true;
        clientCaFile  = certs.ca.cert;
        tlsKeyFile    = certs.${instance}.key;
        tlsCertFile   = certs.${instance}.cert;
        networkPlugin = null;
        clusterDns    = "10.0.0.254";
        extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice";
        inherit kubeconfig;
      };
    };
    networking = {
      firewall = {
        enable = true;
        allowedTCPPorts = [ 53 4194 10250 ];
        allowedUDPPorts = [ 53 ];
        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
      };
    };
    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
    virtualisation.docker.autoPrune.enable = true;
  };

  kubeMaster = {
    services.kubernetes = {
      roles = [ "master" ];
      kubelet.unschedulable = false;
      apiserver = {
        address = masterNode;
        advertiseAddress = masterNode;
        authorizationMode = [ "Node" "RBAC" ];
        securePort = 8443;
        tlsKeyFile   = certs.apiserver.key;
        tlsCertFile  = certs.apiserver.cert;
        clientCaFile = certs.ca.cert;
        kubeletClientCaFile   = certs.ca.cert;
        kubeletClientKeyFile  = certs.apiserver.key;
        kubeletClientCertFile = certs.apiserver.cert;
        serviceAccountKeyFile = certs.apiserver.key;
      };
      scheduler.leaderElect = true;
      controllerManager = {
        leaderElect = true;
        serviceAccountKeyFile = certs.apiserver.key;
        rootCaFile = certs.ca.cert;
        kubeconfig.server = localApiserver;
      };
      scheduler.kubeconfig.server = localApiserver;
      addons.dashboard.enable = true;
      addons.dns.enable = true;
    };
    networking.firewall = {
      allowedTCPPorts = [ 5000 8080 8443 ]; #;4053 ];
      # allowedUDPPorts = [ 4053 ];
    };
    environment.systemPackages = [ pkgs.kubernetes-helm ];
  };

  kubeConfig = instance: {
    services.kubernetes = {
      verbose = false;
      caFile = certs.ca.cert;
      flannel.enable = true;
      clusterCidr = "10.10.0.0/16";
      etcd = {
        servers = etcdEndpoints;
        keyFile  = certs.apiserver.key;
        certFile = certs.apiserver.cert;
        caFile   = certs.ca.cert;
      };
      proxy = {
        kubeconfig = clientConf "kube-proxy";
      };
    };
  };

  nixosConfig = instance: {
    imports = [
      (../nixos/hardware-configuration + "/${instance}.nix")
      ../nixos/configuration.nix
    ];
    services.glusterfs = {
      enable = true;
      tlsSettings = {
        caCert = certs.ca.cert;
        tlsKeyPath = certs.${instance}.key;
        tlsPem = certs.${instance}.cert;
      };
    };
    networking = {
      hostName = instance;
      extraHosts = clusterHosts;
      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
      firewall.allowedTCPPorts = [ 80 443 111 ];
      firewall.allowedUDPPorts = [ 111 24007 24008 ];
    };
    environment.systemPackages = [ pkgs.tshark ];
    # services.dnsmasq.enable = true;
  };

  plain = ip: name: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
      ];
    };

  worker = ip: name: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
        (kubeConfig name)
        (kubeNode name)
      ];
      services.kubernetes.addons.dns.enable = false;
    };

  server = ip: name: etc: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
        (etcdConfig etc)
        (kubeConfig name)
        (kubeNode name)
      ];
      services.kubernetes.addons.dns.enable = false;
    };

  apiserver = ip: name: etc: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
        (etcdConfig etc)
        kubeMaster
        (kubeConfig name)
        (kubeNode name)
      ];
      services.dockerRegistry = {
        enable = true;
        listenAddress = "0.0.0.0";
        extraConfig = {
          REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}";
          REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}";
        };
      };
    };
}