platform/k8s.nix

let
  etcdConfig = name: {
    services.etcd = {
      inherit name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      certFile = ./pki/etcd.pem;
      keyFile = ./pki/etcd-key.pem;
      trustedCaFile = ./pki/ca.pem;
      advertiseClientUrls = [ "https://${name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = [
        "etcd0=https://etcd0:2380"
        "etcd1=https://etcd1:2380"
      ];
    };
    # environment.variables = {
    #   ETCDCTL_CERT_FILE = ./pki + "/${name}.pem";
    #   ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem";
    #   ETCDCTL_CA_FILE = ./pki/ca.pem;
    #   ETCDCTL_PEERS = "https://127.0.0.1:2379";
    # };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
  };

  flannelConfig = node: {
    services.flannel = {
      enable = true;
      network = "10.10.0.0/16";
      iface = "enp2s0";
      etcd = {
        endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
        certFile = ./pki + "/${node}.pem";
        keyFile = ./pki + "/${node}-key.pem";
        caFile = ./pki/ca.pem;
      };
    };
  };

  etcdClient = node:{
    servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
    certFile = ./pki + "/${node}.pem";
    keyFile = ./pki + "/${node}-key.pem";
    caFile = ./pki/ca.pem;
  };

  kubeConfig = node: {
    require = [ (flannelConfig node) ];
    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
    networking.firewall.allowedTCPPorts = [ 10250 ];
    systemd.services.docker.after = [ "flannel.service" ];
    systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
    # services.kubernetes.verbose = true;
  };

  kubeNode = doConfig: node: {
    require = if doConfig then [ (kubeConfig node) ] else [];
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
        server = "https://kubernetes:443";
        caFile = ./pki/ca.pem;
        certFile = ./pki + "/${node}.pem";
        keyFile = ./pki + "/${node}-key.pem";
      };
      kubelet = {
        tlsCertFile = ./pki + "/${node}.pem";
        tlsKeyFile = ./pki + "/${node}-key.pem";
        networkPlugin = null;
        clusterDns = "10.253.18.100";
      };
      etcd = if doConfig then (etcdClient node) else {};
    };
  };

  kubeMaster = node: {
    require = [ (kubeConfig node) (kubeNode false node)];
    services.dockerRegistry = {
      enable = true;
      listenAddress = "0.0.0.0";
    };
    services.kubernetes = {
      roles = [ "master" ];
      apiserver = {
        publicAddress = "0.0.0.0";
        address = "0.0.0.0";
        clientCaFile = ./pki/ca.pem;
        tlsCertFile = ./pki/apiserver.pem;
        tlsKeyFile = ./pki/apiserver-key.pem;
        # kubeletClientCaFile = ./pki/ca.pem;
        # kubeletClientCertFile = ./pki + "/${node}.pem";
        # kubeletClientKeyFile = ./pki + "/${node}-key.pem";
      };
      etcd = (etcdClient node);
      scheduler.leaderElect = true;
      controllerManager.leaderElect = true;
      controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
    };

    networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
    networking.firewall.allowedUDPPorts = [ 53 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

  baseConfig = node: {
    networking.hostName = node;
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
    networking.extraHosts = ''
      10.253.18.100 etcd0 kubernetes
      10.253.18.101 etcd1
    '';
  };
in
{
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      etcd = etcdConfig "etcd0";
      base = baseConfig "k8s0-0";
      master = kubeMaster "k8s0-0";
    in
    {
      deployment.targetHost = "10.253.18.100";
      require = [ base etcd master ];
    };

  k8s0-1 = { config, lib, pkgs, ... }:
    let
      etcd = etcdConfig "etcd1";
      base = baseConfig "k8s0-1";
      node = kubeNode true "k8s0-1";
    in
    {
      deployment.targetHost = "10.253.18.101";
      require = [ base etcd node ];
  };

  k8s0-2 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-2";
      node = kubeNode true "k8s0-2";
    in
    {
      deployment.targetHost = "10.253.18.102";
      require = [ base node ];
  };
}