platform/k8s.nix

with import ./base/pki.nix;
let
  server-cert = mkCert {
      name = "kubernetes";
      csr = csr {
        cn = "kubernetes";
        hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
      };
      profile = "server";
  };

  etcd0-cert = mkCert {
    name = "etcd0";
    csr = csr {
      cn = "etcd0";
      hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
    };
    profile = "peer";
  };

  etcd1-cert = mkCert {
    name = "etcd1";
    csr = csr {
      cn = "etcd1";
      hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
    };
    profile = "peer";
  };

  client-cert = mkCert {
    name = "client";
    csr = csr {
      cn = "client";
      hosts = '''';
    };
    profile = "client";
  };

  server_key = "${server-cert}/cert-key.pem";
  server_cert = "${server-cert}/cert.pem";

  etcd0_key = "${etcd0-cert}/cert-key.pem";
  etcd0_cert = "${etcd0-cert}/cert.pem";

  etcd1_key = "${etcd1-cert}/cert-key.pem";
  etcd1_cert = "${etcd1-cert}/cert.pem";

  client_key = "${client-cert}/cert-key.pem";
  client_cert = "${client-cert}/cert.pem";

  etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ];
  etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ];

  etcdConfig = etcd: {
    services.etcd = {
      name = etcd.name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      keyFile  = "${etcd.key}";
      certFile = "${etcd.cert}";
      trustedCaFile = "${ca_cert}";
      advertiseClientUrls = [ "https://${etcd.name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
      ETCDCTL_KEY_FILE  = "${etcd.key}";
      ETCDCTL_CERT_FILE = "${etcd.cert}";
      ETCDCTL_CA_FILE   = "${ca_cert}";
      ETCDCTL_PEERS     = "https://localhost:2379";
    };
    # networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

  flannelConfig = {
    services.flannel = {
      enable = true;
      network = "10.10.0.0/16";
      iface = "enp2s0";
      etcd = {
        endpoints = etcdServers;
        caFile   = "${ca_cert}";
        keyFile  = "${client_key}";
        certFile = "${client_cert}";
      };
    };
    # networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
  };

  kubeConfig = {
    systemd.services.docker = {
      after = [ "flannel.service" ];
      serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
    };
    virtualisation.docker.extraOptions =
      "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
    services.kubernetes.etcd = {
      servers = etcdServers;
      caFile   = "${ca_cert}";
      keyFile  = "${client_key}";
      certFile = "${client_cert}";
    };
    # services.kubernetes.verbose = true;
  };

  kubeNode = {
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
        server = "https://10.253.18.100:443";
        caFile   = "${ca_cert}";
        keyFile  = "${client_key}";
        certFile = "${client_cert}";
      };
      kubelet = {
        tlsKeyFile  = "${client_key}";
        tlsCertFile = "${client_cert}";
        networkPlugin = null;
        clusterDns = "10.10.21.0";
      };
    };
    networking.firewall = {
      enable = false;
      # allowedTCPPorts = [ 53 10250 8000 8080 ];
      # allowedUDPPorts = [ 53 ];
      # trustedInterfaces = [ "flannel.1" "docker0" ];
      # extraCommands = ''
      #    iptables -P FORWARD ACCEPT
      # '';
    };
  };

  kubeMaster = {
    services.kubernetes = {
      roles = [ "master" ];
      apiserver = {
        publicAddress = "0.0.0.0";
        address = "0.0.0.0";
        clientCaFile = "${ca_cert}";
        tlsKeyFile   = "${server_key}";
        tlsCertFile  = "${server_cert}";
        # serviceAccountKeyFile = "${server_key}";
        # kubeletClientCaFile   = "${ca_cert}";
        # kubeletClientKeyFile  = "${client_key}";
        # kubeletClientCertFile = "${client_cert}";
      };
      scheduler.leaderElect = true;
      controllerManager.leaderElect = true;
      controllerManager.serviceAccountKeyFile = "${server_key}";
    };
    # networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
    # networking.firewall.allowedUDPPorts = [ 53 ];
  };

  baseConfig = node: {
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
    networking.hostName = node;
    networking.extraHosts = ''
      10.253.18.100 etcd0 k8s0-0 kubernetes
      10.253.18.101 etcd1
    '';
    virtualisation.docker.enable = true;
  };

  etcdConf0 = etcdConfig {
    name = "etcd0";
    key = etcd0_key;
    cert = etcd0_cert;
  };

  etcdConf1 = etcdConfig {
    name = "etcd1";
    key = etcd1_key;
    cert = etcd1_cert;
  };

  minion = host: ip: { config, lib, pkgs, ... }:
    let
      inherit host;
      base = baseConfig host;
    in
    {
      deployment.targetHost = ip;
      require = [ base flannelConfig kubeConfig kubeNode ];
  };
in
{
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-0";
      etcd = etcdConf0;
    in
    {
      deployment.targetHost = "10.253.18.100";
      require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ];
      services.dockerRegistry = {
        enable = true;
        listenAddress = "0.0.0.0";
      };
    };

  k8s0-1 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-1";
      etcd = etcdConf1;
    in
    {
      deployment.targetHost = "10.253.18.101";
      require = [ base etcd flannelConfig kubeConfig kubeNode ];
  };

  k8s0-2 = minion "k8s0-2" "10.253.18.102";
}