platform/k8s.nix

with import ./base/pki.nix;
let
  server-cert = mkCert {
      name = "kubernetes";
      csr = csr {
        cn = "kubernetes";
        hosts = ''"kubernetes", "k8s0-0", "10.253.18.100"'';
      };
      profile = "server";
  };

  etcd0-cert = mkCert {
    name = "etcd0";
    csr = csr {
      cn = "etcd0";
      hosts = ''"etcd0", "10.253.18.100"'';
    };
    profile = "peer";
  };

  etcd1-cert = mkCert {
    name = "etcd1";
    csr = csr {
      cn = "etcd1";
      hosts = ''"etcd1", "10.253.18.101"'';
    };
    profile = "peer";
  };

  client-cert = mkCert {
    name = "client";
    csr = csr {
      cn = "client";
      hosts = '''';
    };
    profile = "client";
  };

  server_key = "${server-cert}/cert-key.pem";
  server_cert = "${server-cert}/cert.pem";

  etcd0_key = "${etcd0-cert}/cert-key.pem";
  etcd0_cert = "${etcd0-cert}/cert.pem";

  etcd1_key = "${etcd1-cert}/cert-key.pem";
  etcd1_cert = "${etcd1-cert}/cert.pem";

  client_key = "${client-cert}/cert-key.pem";
  client_cert = "${client-cert}/cert.pem";

  etcdCluster = [
    "etcd0=https://etcd0:2380"
    "etcd1=https://etcd1:2380"
  ];

  etcdConfig = etcd: {
    services.etcd = {
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      keyFile  = "${etcd.key}";
      certFile = "${etcd.cert}";
      trustedCaFile = "${ca_cert}";
      advertiseClientUrls = [ "https://${etcd.name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
      ETCDCTL_KEY_FILE  = "${etcd.key}";
      ETCDCTL_CERT_FILE = "${etcd.cert}";
      ETCDCTL_CA_FILE   = "${ca_cert}";
      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
    };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
  };

  flannelConfig = {
    services.flannel = {
      enable = true;
      network = "10.10.0.0/16";
      iface = "enp2s0";
      etcd = {
        endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
        caFile   = "${ca_cert}";
        keyFile  = "${client_key}";
        certFile = "${client_cert}";
      };
    };
  };

  kubeConfig = {
    require = [ flannelConfig ];
    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
    networking.firewall.allowedTCPPorts = [ 10250 ];
    systemd.services.docker = {
      after = [ "flannel.service" ];
      serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
    };
    virtualisation.docker.extraOptions =
      "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
    services.kubernetes.etcd = {
      servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
      caFile   = "${ca_cert}";
      keyFile  = "${client_key}";
      certFile = "${client_cert}";
    };
    # services.kubernetes.verbose = true;
  };

  kubeNode = {
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
        server = "https://kubernetes:443";
        caFile   = "${ca_cert}";
        keyFile  = "${client_key}";
        certFile = "${client_cert}";
      };
      kubelet = {
        tlsKeyFile  = "${client_key}";
        tlsCertFile = "${client_cert}";
        extraOpts = "--client-ca-file=${ca_cert}";
        networkPlugin = null;
        clusterDns = "kubernetes";
      };
    };
  };

  kubeMaster = {
    services.dockerRegistry = {
      enable = true;
      listenAddress = "0.0.0.0";
    };
    services.kubernetes = {
      roles = [ "master" ];
      apiserver = {
        publicAddress = "0.0.0.0";
        address = "0.0.0.0";
        clientCaFile = "${ca_cert}";
        tlsKeyFile   = "${server_key}";
        tlsCertFile  = "${server_cert}";
        # kubeletClientCaFile   = "${ca_cert}";
        # kubeletClientKeyFile  = "${server_key}";
        # kubeletClientCertFile = "${server_cert}";
      };
      scheduler.leaderElect = true;
      controllerManager.leaderElect = true;
      controllerManager.serviceAccountKeyFile = "${server_key}";
    };

    networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
    networking.firewall.allowedUDPPorts = [ 53 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

  baseConfig = node: {
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
    require = [ kubeConfig ];
    networking.hostName = node;
    networking.extraHosts = ''
      10.253.18.100 etcd0 kubernetes
      10.253.18.101 etcd1
    '';
  };
in
{
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      host = "k8s0-0";
      base = baseConfig host;
      etcd = etcdConfig {
        name = "etcd0";
        key = etcd0_key;
        cert = etcd0_cert;
      };
    in
    {
      deployment.targetHost = "10.253.18.100";
      require = [ base etcd kubeMaster kubeNode ];
    };

  k8s0-1 = { config, lib, pkgs, ... }:
    let
      host = "k8s0-1";
      base = baseConfig host;
      etcd = etcdConfig {
        name = "etcd1";
        key = etcd1_key;
        cert = etcd1_cert;
      };
    in
    {
      deployment.targetHost = "10.253.18.101";
      require = [ base etcd kubeNode ];
  };

  k8s0-2 = { config, lib, pkgs, ... }:
    let
      host = "k8s0-2";
      base = baseConfig host;
    in
    {
      deployment.targetHost = "10.253.18.102";
      require = [ base kubeNode ];
  };
}