platform/k8s.nix

with import ./certs.nix;
let
  pkgs = import <nixpkgs> {};

  etcdServers = [ "etcd0" "etcd1" "etcd2" ];
  # etcdServers = [ "k8s0-0" "k8s0-1" "k8s0-2" ];
  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdServers;

  etcdConfig = name: {
    services.etcd = {
      inherit name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      keyFile  = etcd_key;
      certFile = etcd_cert;
      trustedCaFile = ca_pem;
      advertiseClientUrls = [ "https://${name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
      ETCDCTL_KEY_FILE  = "${etcd_client_key}";
      ETCDCTL_CERT_FILE = "${etcd_client_cert}";
      ETCDCTL_CA_FILE   = "${ca_pem}";
      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
    };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

  kubeConfig = {
    services.flannel = {
      enable = true;
      network = "10.10.0.0/16";
      iface = "ens32";
      etcd = {
        endpoints = etcdEndpoints;
        keyFile  = etcd_client_key;
        certFile = etcd_client_cert;
        caFile   = ca_pem;
      };
    };
    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
    systemd.services.docker = {
      after = [ "flannel.service" ];
      serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
    };
    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU";
    services.kubernetes.etcd = {
      servers = etcdEndpoints;
      keyFile  = etcd_client_key;
      certFile = etcd_client_cert;
      caFile   = ca_pem;
    };
    # services.kubernetes.verbose = true;
  };

  kubeNode = {
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
        server = "https://10.253.18.100:4443";
        keyFile  = worker_key;
        certFile = worker_cert;
        caFile   = ca_pem;
      };
      kubelet = {
        tlsKeyFile  = worker_key;
        tlsCertFile = worker_cert;
        networkPlugin = null;
        clusterDns = "10.253.18.100";
      };
    };
    networking = {
      firewall = {
        enable = true;
        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
        allowedTCPPorts = [ 53 10250 ];
        allowedUDPPorts = [ 53 ];
        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
      };
    };
  };

  kubeMaster = {
    services.kubernetes = {
      roles = [ "master" ];
      apiserver = {
        address = "0.0.0.0";
        publicAddress = "0.0.0.0";
        advertiseAddress = "10.253.18.100";
        securePort = 4443;
        tlsKeyFile   = apiserver_key;
        tlsCertFile  = apiserver_cert;
        clientCaFile = ca_pem;
        kubeletClientCaFile   = ca_pem;
        kubeletClientKeyFile  = worker_key;
        kubeletClientCertFile = worker_cert;
        serviceAccountKeyFile = apiserver_key;
      };
      scheduler.leaderElect = true;
      controllerManager.leaderElect = true;
      controllerManager.serviceAccountKeyFile = apiserver_key;
      controllerManager.rootCaFile = ca_pem;
      dns.enable = true;
      dns.port = 4053;
    };
    networking.firewall = {
      allowedTCPPorts = [ 5000 8080 4443 4053 ];
      allowedUDPPorts = [ 4053 ];
    };
    environment.systemPackages = [ pkgs.kubernetes-helm ];
  };

  baseConfig = node: {
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
    networking = {
      hostName = node;
      extraHosts = ''
        10.253.18.100 etcd0 kubernetes
        10.253.18.101 etcd1
        10.253.18.102 etcd2
      '';
      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
      firewall.allowedTCPPorts = [ 80 443 ];
    };
    services.dnsmasq.enable = true;
    services.dnsmasq.servers = [
      "/cluster.local/10.253.18.100#4053"
    ];
  };

  minion = host: ip: { config, lib, pkgs, ... }:
    let
      inherit host;
      base = baseConfig host;
    in
    {
      deployment.targetHost = ip;
      require = [ base kubeConfig kubeNode ];
      services.kubernetes.dns.enable = false;
    };
in
{
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-0";
      etcd = etcdConfig "etcd0";
    in
    {
      deployment.targetHost = "10.253.18.100";
      require = [ base etcd kubeConfig kubeMaster kubeNode ];
      services.dockerRegistry = {
        enable = true;
        listenAddress = "0.0.0.0";
      };
    };

  k8s0-1 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-1";
      etcd = etcdConfig "etcd1";
    in
    {
      deployment.targetHost = "10.253.18.101";
      require = [ base etcd kubeConfig kubeNode ];
      services.kubernetes.dns.enable = false;
    };

  k8s0-2 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-2";
      etcd = etcdConfig "etcd2";
    in
    {
      deployment.targetHost = "10.253.18.102";
      require = [ base etcd kubeConfig kubeNode ];
      services.kubernetes.dns.enable = false;
    };
}