with import ./certs.nix; let pkgs = import {}; etcdServers = [ "etcd0" "etcd1" "etcd2" ]; # etcdServers = [ "k8s0-0" "k8s0-1" "k8s0-2" ]; etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers; etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers; etcdConfig = name: { services.etcd = { inherit name; enable = true; listenClientUrls = ["https://0.0.0.0:2379"]; listenPeerUrls = ["https://0.0.0.0:2380"]; peerClientCertAuth = true; keyFile = etcd_key; certFile = etcd_cert; trustedCaFile = ca_pem; advertiseClientUrls = [ "https://${name}:2379" ]; initialAdvertisePeerUrls = [ "https://${name}:2380" ]; initialCluster = etcdCluster; }; environment.variables = { ETCDCTL_KEY_FILE = "${etcd_client_key}"; ETCDCTL_CERT_FILE = "${etcd_client_cert}"; ETCDCTL_CA_FILE = "${ca_pem}"; ETCDCTL_PEERS = "https://127.0.0.1:2379"; }; networking.firewall.allowedTCPPorts = [ 2379 2380 ]; systemd.services.flannel.after = [ "etcd.service" ]; }; kubeConfig = { services.flannel = { enable = true; network = "10.10.0.0/16"; iface = "ens32"; etcd = { endpoints = etcdEndpoints; keyFile = etcd_client_key; certFile = etcd_client_cert; caFile = ca_pem; }; }; networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN systemd.services.docker = { after = [ "flannel.service" ]; serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; }; virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU"; services.kubernetes.etcd = { servers = etcdEndpoints; keyFile = etcd_client_key; certFile = etcd_client_cert; caFile = ca_pem; }; # services.kubernetes.verbose = true; }; kubeNode = { services.kubernetes = { roles = [ "node" ]; kubeconfig = { server = "https://10.253.18.100:4443"; keyFile = worker_key; certFile = worker_cert; caFile = ca_pem; }; kubelet = { tlsKeyFile = worker_key; tlsCertFile = worker_cert; networkPlugin = null; clusterDns = "10.253.18.100"; }; }; networking = { firewall = { enable = true; # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; allowedTCPPorts = [ 53 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; }; kubeMaster = { services.kubernetes = { roles = [ "master" ]; apiserver = { address = "0.0.0.0"; publicAddress = "0.0.0.0"; advertiseAddress = "10.253.18.100"; securePort = 4443; tlsKeyFile = apiserver_key; tlsCertFile = apiserver_cert; clientCaFile = ca_pem; kubeletClientCaFile = ca_pem; kubeletClientKeyFile = worker_key; kubeletClientCertFile = worker_cert; serviceAccountKeyFile = apiserver_key; }; scheduler.leaderElect = true; controllerManager.leaderElect = true; controllerManager.serviceAccountKeyFile = apiserver_key; controllerManager.rootCaFile = ca_pem; dns.enable = true; dns.port = 4053; }; networking.firewall = { allowedTCPPorts = [ 5000 8080 4443 4053 ]; allowedUDPPorts = [ 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm ]; }; baseConfig = node: { imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; networking = { hostName = node; extraHosts = '' 10.253.18.100 etcd0 kubernetes 10.253.18.101 etcd1 10.253.18.102 etcd2 ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 ]; }; services.dnsmasq.enable = true; services.dnsmasq.servers = [ "/cluster.local/10.253.18.100#4053" ]; }; minion = host: ip: { config, lib, pkgs, ... }: let inherit host; base = baseConfig host; in { deployment.targetHost = ip; require = [ base kubeConfig kubeNode ]; services.kubernetes.dns.enable = false; }; in { k8s0-0 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-0"; etcd = etcdConfig "etcd0"; in { deployment.targetHost = "10.253.18.100"; require = [ base etcd kubeConfig kubeMaster kubeNode ]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; }; }; k8s0-1 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-1"; etcd = etcdConfig "etcd1"; in { deployment.targetHost = "10.253.18.101"; require = [ base etcd kubeConfig kubeNode ]; services.kubernetes.dns.enable = false; }; k8s0-2 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-2"; etcd = etcdConfig "etcd2"; in { deployment.targetHost = "10.253.18.102"; require = [ base etcd kubeConfig kubeNode ]; services.kubernetes.dns.enable = false; }; }