platform/lib/k8s.nix

{ pkgs, masterAddress, etcdNodes, clusterHosts, certs, ...}:
let
  kubeApiserver = "https://${masterAddress}:8443";
  localApiserver = "http://127.0.0.1:8080";
  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
in
rec {
  etcdConfig = name: {
    services.etcd = {
      inherit name;
      enable = true;
      listenClientUrls = [ "https://0.0.0.0:2379" ];
      listenPeerUrls = [ "https://0.0.0.0:2380" ];
      peerClientCertAuth = true;
      keyFile  = certs.etcd.key;
      certFile = certs.etcd.cert;
      trustedCaFile = certs.ca.cert;
      advertiseClientUrls = [ "https://${name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
      ETCDCTL_KEY_FILE  = "${certs.admin.key}";
      ETCDCTL_CERT_FILE = "${certs.admin.cert}";
      ETCDCTL_CA_FILE   = "${certs.ca.cert}";
      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
    };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

  clientConf = instance: {
    server = kubeApiserver;
    keyFile = certs.${instance}.key;
    certFile = certs.${instance}.cert;
    caFile = certs.ca.cert;
  };

  kubeNode = instance: {
    services.kubernetes = rec {
      inherit masterAddress;
      roles = [ "node" ];
      kubeconfig = clientConf instance;
      kubelet = {
        enable = true;
        clientCaFile  = certs.ca.cert;
        tlsKeyFile    = certs.${instance}.key;
        tlsCertFile   = certs.${instance}.cert;
        networkPlugin = null;
        clusterDns    = "10.0.0.254";
        extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice";
        inherit kubeconfig;
      };
    };
    networking = {
      firewall = {
        enable = true;
        allowedTCPPorts = [ 4194 10250 ];
        # allowedUDPPorts = [ 53 ];
        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
      };
    };
    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
    virtualisation.docker.autoPrune.enable = true;
  };

  kubeMaster = {
    services.kubernetes = {
      roles = [ "master" ];
      kubelet.unschedulable = false;
      apiserver = {
        bindAddress = "0.0.0.0"; #masterAddress;
        advertiseAddress = masterAddress;
        authorizationMode = [ "Node" "RBAC" ];
        securePort = 8443;
        tlsKeyFile   = certs.apiserver.key;
        tlsCertFile  = certs.apiserver.cert;
        clientCaFile = certs.ca.cert;
        kubeletClientCaFile   = certs.ca.cert;
        kubeletClientKeyFile  = certs.apiserver.key;
        kubeletClientCertFile = certs.apiserver.cert;
        serviceAccountKeyFile = certs.apiserver.key;
        etcd = {
          servers = etcdEndpoints;
          keyFile  = certs.apiserver.key;
          certFile = certs.apiserver.cert;
          caFile   = certs.ca.cert;
        };
      };
      scheduler.leaderElect = true;
      controllerManager = {
        leaderElect = true;
        serviceAccountKeyFile = certs.apiserver.key;
        # rootCaFile = certs.ca.cert;
        kubeconfig.server = localApiserver;
      };
      scheduler.kubeconfig.server = localApiserver;
      addons.dns.enable = true;
      addons.dns.reconcileMode = "EnsureExists";
      addons.dashboard = rec {
      	enable = true;
      	version = "v1.10.0";
        rbac.enable = true;
        rbac.clusterAdmin = true;
        # tokenTtl = 0;
     	image = {
          imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
          imageDigest = "sha256:1d2e1229a918f4bc38b5a3f9f5f11302b3e71f8397b492afac7f273a0008776a";
          finalImageTag = version;
          sha256 = "10qkqqhzkr0bcv0dlf8nq069h190pw6zjj1l5s5g438g80v8639j";
        };
      };
    };
    networking.firewall = {
      allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
      allowedUDPPorts = [ 53 4053 ];
    };
    environment.systemPackages = [ pkgs.kubernetes-helm ];
  };

  kubeConfig = instance: {
    services.kubernetes = {
      # caFile = certs.ca.cert;
      flannel.enable = true;
      clusterCidr = "10.10.0.0/16";
      proxy = {
        kubeconfig = clientConf "kube-proxy";
      };
    };
  };

  nixosConfig = instance: {
    imports = [
      (../nixos/hardware-configuration + "/${instance}.nix")
      ../nixos/configuration.nix
    ];
    security.pki.certificateFiles = [
      certs.ca.cert
    ];
    services.glusterfs = {
      enable = true;
      tlsSettings = {
        caCert = certs.ca.cert;
        tlsKeyPath = certs.${instance}.key;
        tlsPem = certs.${instance}.cert;
      };
    };
    networking = {
      hostName = instance;
      extraHosts = clusterHosts;
      # nameservers = [ masterAddress ];
      # dhcpcd.extraConfig = ''
      #  static domain_name_servers=${masterAddress}
      # '';
      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
      firewall.allowedTCPPorts = [ 80 443 111 ];
      firewall.allowedUDPPorts = [ 111 24007 24008 ];
    };
  };

  plain = ip: name: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
      ];
    };

  worker = ip: name: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
        (kubeConfig name)
        (kubeNode name)
      ];
      services.kubernetes.addons.dns.enable = false;
    };

  server = ip: name: etc: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
        (etcdConfig etc)
        (kubeConfig name)
        (kubeNode name)
      ];
      services.kubernetes.addons.dns.enable = false;
    };

  apiserver = ip: name: etc: { config, lib, pkgs, ... }:
    {
      deployment.targetHost = ip;
      require = [
        (nixosConfig name)
        (etcdConfig etc)
        kubeMaster
        (kubeConfig name)
        (kubeNode name)
      ];
    };
}