platform/k8s.nix

let
  etcdConfig = name: {
    services.etcd = {
      inherit name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      certFile = ./pki/etcd.pem;
      keyFile = ./pki/etcd-key.pem;
      trustedCaFile = ./pki/ca.pem;
      advertiseClientUrls = [ "https://${name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = [
        "etcd0=https://etcd0:2380"
        "etcd1=https://etcd1:2380"
      ];
    };
    # environment.variables = {
    #   ETCDCTL_CERT_FILE = ./pki + "/${name}.pem";
    #   ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem";
    #   ETCDCTL_CA_FILE = ./pki/ca.pem;
    #   ETCDCTL_PEERS = "https://127.0.0.1:2379";
    # };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
  };

  flannelConfig = node: {
    services.flannel = {
      enable = true;
      network = "10.10.0.0/16";
      iface = "enp2s0";
      etcd = {
        endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
        certFile = ./pki + "/${node}.pem";
        keyFile = ./pki + "/${node}-key.pem";
        caFile = ./pki/ca.pem;
      };
    };
  };

  kubeConfig = node: {
    require = [ (flannelConfig node) ];
    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
    networking.firewall.allowedTCPPorts = [ 10250 ];
    systemd.services.docker.after = [ "flannel.service" ];
    systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
    # services.kubernetes.verbose = true;
    services.kubernetes.etcd = {
      servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
      certFile = ./pki + "/${node}.pem";
      keyFile = ./pki + "/${node}-key.pem";
      caFile = ./pki/ca.pem;
    };
  };

  kubeNode = doConfig: node: {
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
        server = "https://kubernetes:443";
        caFile = ./pki/ca.pem;
        certFile = ./pki + "/${node}.pem";
        keyFile = ./pki + "/${node}-key.pem";
      };
      kubelet = {
        tlsCertFile = ./pki + "/${node}.pem";
        tlsKeyFile = ./pki + "/${node}-key.pem";
        networkPlugin = null;
        clusterDns = "kubernetes";
      };
    };
  };

  kubeMaster = node: {
    services.dockerRegistry = {
      enable = true;
      listenAddress = "0.0.0.0";
    };
    services.kubernetes = {
      roles = [ "master" ];
      apiserver = {
        publicAddress = "0.0.0.0";
        address = "0.0.0.0";
        clientCaFile = ./pki/ca.pem;
        tlsCertFile = ./pki/apiserver.pem;
        tlsKeyFile = ./pki/apiserver-key.pem;
        # kubeletClientCaFile = ./pki/ca.pem;
        # kubeletClientCertFile = ./pki + "/${node}.pem";
        # kubeletClientKeyFile = ./pki + "/${node}-key.pem";
      };
      scheduler.leaderElect = true;
      controllerManager.leaderElect = true;
      controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
    };

    networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
    networking.firewall.allowedUDPPorts = [ 53 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

  baseConfig = node: {
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
    require = [ (kubeConfig node) ];
    networking.hostName = node;
    networking.extraHosts = ''
      10.253.18.100 etcd0 kubernetes
      10.253.18.101 etcd1
    '';
  };
in
{
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      host = "k8s0-0";
      etcd = etcdConfig "etcd0";
      base = baseConfig host;
      master = kubeMaster host;
      node = kubeNode true host;
    in
    {
      deployment.targetHost = "10.253.18.100";
      require = [ base etcd master node ];
    };

  k8s0-1 = { config, lib, pkgs, ... }:
    let
      host = "k8s0-1";
      etcd = etcdConfig "etcd1";
      base = baseConfig host;
      node = kubeNode true host;
    in
    {
      deployment.targetHost = "10.253.18.101";
      require = [ base etcd node ];
  };

  k8s0-2 = { config, lib, pkgs, ... }:
    let
      host = "k8s0-2";
      base = baseConfig host;
      node = kubeNode true host;
    in
    {
      deployment.targetHost = "10.253.18.102";
      require = [ base node ];
  };
}