platform/k8s.nix

with import ./certs.nix;
let
  pkgs = import <nixpkgs> {};

  kube_apiserver = "https://10.253.18.100:443";
  etcdServers = [ "etcd0" "etcd1" "etcd2" ];
  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdServers;

  etcdConfig = name: {
    services.etcd = {
      inherit name;
      enable = true;
      listenClientUrls = ["https://0.0.0.0:2379"];
      listenPeerUrls = ["https://0.0.0.0:2380"];
      peerClientCertAuth = true;
      keyFile  = etcd_key;
      certFile = etcd_cert;
      trustedCaFile = ca_pem;
      advertiseClientUrls = [ "https://${name}:2379" ];
      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
      initialCluster = etcdCluster;
    };
    environment.variables = {
      ETCDCTL_KEY_FILE  = "${etcd_client_key}";
      ETCDCTL_CERT_FILE = "${etcd_client_cert}";
      ETCDCTL_CA_FILE   = "${ca_pem}";
      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
    };
    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    systemd.services.flannel.after = [ "etcd.service" ];
  };

   kubeconfig = {
    caFile = ca_pem;
    keyFile = worker_key;
    certFile = worker_cert;
    server = kube_apiserver;
  };

  kubeNode = {
    services.kubernetes = {
      roles = [ "node" ];
      kubeconfig = {
        server   = kube_apiserver;
        keyFile  = worker_key;
        certFile = worker_cert;
        caFile   = ca_pem;
      };
      kubelet = {
        enable = true;
        clientCaFile  = ca_pem;
        tlsKeyFile    = worker_key;
        tlsCertFile   = worker_cert;
        networkPlugin = null;
        # clusterDns    = "10.253.18.100";
        clusterDns    = "10.0.0.254";
        inherit kubeconfig;
      };
    };
    networking = {
      firewall = {
        enable = true;
        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
        allowedTCPPorts = [ 53 4194 10250 ];
        allowedUDPPorts = [ 53 ];
        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
      };
    };
    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
  };

  kubeMaster = {
    services.kubernetes = {
      roles = [ "master" ];
      kubelet.unschedulable = true;
      apiserver = {
        address = "0.0.0.0";
        publicAddress = "0.0.0.0";
        advertiseAddress = "10.253.18.100";
        securePort = 443;
        tlsKeyFile   = apiserver_key;
        tlsCertFile  = apiserver_cert;
        clientCaFile = ca_pem;
        kubeletClientCaFile   = ca_pem;
        kubeletClientKeyFile  = worker_key;
        kubeletClientCertFile = worker_cert;
        serviceAccountKeyFile = apiserver_key;
      };
      scheduler.leaderElect = true;
      controllerManager = {
        leaderElect = true;
        serviceAccountKeyFile = apiserver_key;
        rootCaFile = ca_pem;
        inherit kubeconfig;
      };
      addons.dashboard.enable = true;
      addons.dns.enable = true;
    };
    networking.firewall = {
      allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
      # allowedUDPPorts = [ 4053 ];
    };
    environment.systemPackages = [ pkgs.kubernetes-helm ];
  };

  kubeConfig = {
    services.kubernetes = {
      verbose = false;
      caFile = ca_pem;
      flannel.enable = true;
      clusterCidr = "10.10.0.0/16";
      etcd = {
        servers = etcdEndpoints;
        keyFile  = etcd_client_key;
        certFile = etcd_client_cert;
        caFile   = ca_pem;
      };
      proxy = {
        inherit kubeconfig;
      };
    };
  };

  baseConfig = node: {
    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
    networking = {
      hostName = node;
      extraHosts = ''
        10.253.18.100 etcd0 kubernetes
        10.253.18.101 etcd1
        10.253.18.102 etcd2
      '';
      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
      firewall.allowedTCPPorts = [ 80 443 ];
    };
    services.dnsmasq.enable = true;
    services.dnsmasq.servers = [
      "/cluster.local/10.0.0.254#53"
    ];
  };

  minion = host: ip: { config, lib, pkgs, ... }:
    let
      inherit host;
      base = baseConfig host;
    in
    {
      deployment.targetHost = ip;
      require = [ base kubeConfig kubeNode ];
      services.kubernetes.addons.dns.enable = false;
    };
in
{
  k8s0-0 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-0";
      etcd = etcdConfig "etcd0";
    in
    {
      deployment.targetHost = "10.253.18.100";
      require = [ base etcd kubeConfig kubeMaster kubeNode ];
      services.dockerRegistry = {
        enable = true;
        listenAddress = "0.0.0.0";
        extraConfig = {
          REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver_cert}";
          REGISTRY_HTTP_TLS_KEY = "${apiserver_key}";
        };
      };
    };

  k8s0-1 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-1";
      etcd = etcdConfig "etcd1";
    in
    {
      deployment.targetHost = "10.253.18.101";
      require = [ base etcd kubeConfig kubeNode ];
      services.kubernetes.addons.dns.enable = false;
    };

  k8s0-2 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "k8s0-2";
      etcd = etcdConfig "etcd2";
    in
    {
      deployment.targetHost = "10.253.18.102";
      require = [ base etcd kubeConfig kubeNode ];
      services.kubernetes.addons.dns.enable = false;
    };

  git01 = { config, lib, pkgs, ... }:
    let
      base = baseConfig "git01";
    in
    {
      deployment.targetHost = "10.253.18.103";
      require = [ base kubeConfig kubeNode ];
      services.kubernetes.addons.dns.enable = false;
      services.nfs.server = {
        enable=true;
        exports= ''
          /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
        '';
      };
      networking.firewall.allowedTCPPorts = [ 111 2049 ];
      networking.firewall.allowedUDPPorts = [ 111 2049 ];
    };
}