{ pkgs, masterNode, etcdNodes, clusterHosts, certs, ...}: let kubeApiserver = "https://${masterNode}:8443"; localApiserver = "http://127.0.0.1:8080"; etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes; etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes; in rec { etcdConfig = name: { services.etcd = { inherit name; enable = true; listenClientUrls = ["https://0.0.0.0:2379"]; listenPeerUrls = ["https://0.0.0.0:2380"]; peerClientCertAuth = true; keyFile = certs.etcd.key; certFile = certs.etcd.cert; trustedCaFile = certs.ca.cert; advertiseClientUrls = [ "https://${name}:2379" ]; initialAdvertisePeerUrls = [ "https://${name}:2380" ]; initialCluster = etcdCluster; }; environment.variables = { ETCDCTL_KEY_FILE = "${certs.admin.key}"; ETCDCTL_CERT_FILE = "${certs.admin.cert}"; ETCDCTL_CA_FILE = "${certs.ca.cert}"; ETCDCTL_PEERS = "https://127.0.0.1:2379"; }; networking.firewall.allowedTCPPorts = [ 2379 2380 ]; systemd.services.flannel.after = [ "etcd.service" ]; }; clientConf = instance: { server = kubeApiserver; keyFile = certs.${instance}.key; certFile = certs.${instance}.cert; caFile = certs.ca.cert; }; kubeNode = instance: { services.kubernetes = rec { roles = [ "node" ]; kubeconfig = clientConf instance; kubelet = { enable = true; clientCaFile = certs.ca.cert; tlsKeyFile = certs.${instance}.key; tlsCertFile = certs.${instance}.cert; networkPlugin = null; clusterDns = "10.0.0.254"; extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice"; inherit kubeconfig; }; }; networking = { firewall = { enable = true; allowedTCPPorts = [ 4194 10250 ]; # allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; }; kubeMaster = { services.kubernetes = { roles = [ "master" ]; kubelet.unschedulable = false; apiserver = { bindAddress = "0.0.0.0"; #masterNode; advertiseAddress = masterNode; authorizationMode = [ "Node" "RBAC" ]; securePort = 8443; tlsKeyFile = certs.apiserver.key; tlsCertFile = certs.apiserver.cert; clientCaFile = certs.ca.cert; kubeletClientCaFile = certs.ca.cert; kubeletClientKeyFile = certs.apiserver.key; kubeletClientCertFile = certs.apiserver.cert; serviceAccountKeyFile = certs.apiserver.key; }; scheduler.leaderElect = true; controllerManager = { leaderElect = true; serviceAccountKeyFile = certs.apiserver.key; rootCaFile = certs.ca.cert; kubeconfig.server = localApiserver; }; scheduler.kubeconfig.server = localApiserver; addons.dns.enable = true; addons.dns.reconcileMode = "EnsureExists"; addons.dashboard = rec { enable = true; version = "v1.10.0"; rbac.enable = true; rbac.clusterAdmin = true; tokenTtl = 0; image = { imageName = "k8s.gcr.io/kubernetes-dashboard-amd64"; imageDigest = "sha256:1d2e1229a918f4bc38b5a3f9f5f11302b3e71f8397b492afac7f273a0008776a"; finalImageTag = version; sha256 = "10qkqqhzkr0bcv0dlf8nq069h190pw6zjj1l5s5g438g80v8639j"; }; }; }; networking.firewall = { allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm ]; }; kubeConfig = instance: { services.kubernetes = { verbose = false; caFile = certs.ca.cert; flannel.enable = true; clusterCidr = "10.10.0.0/16"; etcd = { servers = etcdEndpoints; keyFile = certs.apiserver.key; certFile = certs.apiserver.cert; caFile = certs.ca.cert; }; proxy = { kubeconfig = clientConf "kube-proxy"; }; }; }; nixosConfig = instance: { imports = [ (../nixos/hardware-configuration + "/${instance}.nix") ../nixos/configuration.nix ]; security.pki.certificateFiles = [ certs.ca.cert ]; services.glusterfs = { enable = true; tlsSettings = { caCert = certs.ca.cert; tlsKeyPath = certs.${instance}.key; tlsPem = certs.${instance}.cert; }; }; networking = { hostName = instance; extraHosts = clusterHosts; # nameservers = [ masterNode ]; # dhcpcd.extraConfig = '' # static domain_name_servers=${masterNode} # ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; }; plain = ip: name: { config, lib, pkgs, ... }: { deployment.targetHost = ip; require = [ (nixosConfig name) ]; }; worker = ip: name: { config, lib, pkgs, ... }: { deployment.targetHost = ip; require = [ (nixosConfig name) (kubeConfig name) (kubeNode name) ]; services.kubernetes.addons.dns.enable = false; }; server = ip: name: etc: { config, lib, pkgs, ... }: { deployment.targetHost = ip; require = [ (nixosConfig name) (etcdConfig etc) (kubeConfig name) (kubeNode name) ]; services.kubernetes.addons.dns.enable = false; }; apiserver = ip: name: etc: { config, lib, pkgs, ... }: { deployment.targetHost = ip; require = [ (nixosConfig name) (etcdConfig etc) kubeMaster (kubeConfig name) (kubeNode name) ]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; enableDelete = true; enableGarbageCollect = true; extraConfig = { REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}"; REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}"; }; }; }; }