let etcdConfig = name: { services.etcd = { inherit name; enable = true; listenClientUrls = ["https://0.0.0.0:2379"]; listenPeerUrls = ["https://0.0.0.0:2380"]; peerClientCertAuth = true; certFile = ./pki/etcd.pem; keyFile = ./pki/etcd-key.pem; trustedCaFile = ./pki/ca.pem; advertiseClientUrls = [ "https://${name}:2379" ]; initialAdvertisePeerUrls = [ "https://${name}:2380" ]; initialCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ]; }; # environment.variables = { # ETCDCTL_CERT_FILE = ./pki + "/${name}.pem"; # ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem"; # ETCDCTL_CA_FILE = ./pki/ca.pem; # ETCDCTL_PEERS = "https://127.0.0.1:2379"; # }; networking.firewall.allowedTCPPorts = [ 2379 2380 ]; }; flannelConfig = node: { services.flannel = { enable = true; network = "10.10.0.0/16"; iface = "enp2s0"; etcd = { endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ]; certFile = ./pki + "/${node}.pem"; keyFile = ./pki + "/${node}-key.pem"; caFile = ./pki/ca.pem; }; }; }; etcdClient = node:{ servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; certFile = ./pki + "/${node}.pem"; keyFile = ./pki + "/${node}-key.pem"; caFile = ./pki/ca.pem; }; kubeConfig = node: { require = [ (flannelConfig node) ]; networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN networking.firewall.allowedTCPPorts = [ 10250 ]; systemd.services.docker.after = [ "flannel.service" ]; systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; # services.kubernetes.verbose = true; }; kubeNode = doConfig: node: { require = if doConfig then [ (kubeConfig node) ] else []; services.kubernetes = { roles = [ "node" ]; kubeconfig = { server = "https://kubernetes:443"; caFile = ./pki/ca.pem; certFile = ./pki + "/${node}.pem"; keyFile = ./pki + "/${node}-key.pem"; }; kubelet = { tlsCertFile = ./pki + "/${node}.pem"; tlsKeyFile = ./pki + "/${node}-key.pem"; networkPlugin = null; clusterDns = "10.253.18.100"; }; etcd = if doConfig then (etcdClient node) else {}; }; }; kubeMaster = node: { require = [ (kubeConfig node) (kubeNode false node)]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; }; services.kubernetes = { roles = [ "master" ]; apiserver = { publicAddress = "0.0.0.0"; address = "0.0.0.0"; clientCaFile = ./pki/ca.pem; tlsCertFile = ./pki/apiserver.pem; tlsKeyFile = ./pki/apiserver-key.pem; # kubeletClientCaFile = ./pki/ca.pem; # kubeletClientCertFile = ./pki + "/${node}.pem"; # kubeletClientKeyFile = ./pki + "/${node}-key.pem"; }; etcd = (etcdClient node); scheduler.leaderElect = true; controllerManager.leaderElect = true; controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem; }; networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; networking.firewall.allowedUDPPorts = [ 53 ]; systemd.services.flannel.after = [ "etcd.service" ]; }; baseConfig = node: { networking.hostName = node; imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; networking.extraHosts = '' 10.253.18.100 etcd0 kubernetes 10.253.18.101 etcd1 ''; }; in { k8s0-0 = { config, lib, pkgs, ... }: let etcd = etcdConfig "etcd0"; base = baseConfig "k8s0-0"; master = kubeMaster "k8s0-0"; in { deployment.targetHost = "10.253.18.100"; require = [ base etcd master ]; }; k8s0-1 = { config, lib, pkgs, ... }: let etcd = etcdConfig "etcd1"; base = baseConfig "k8s0-1"; node = kubeNode true "k8s0-1"; in { deployment.targetHost = "10.253.18.101"; require = [ base etcd node ]; }; k8s0-2 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-2"; node = kubeNode true "k8s0-2"; in { deployment.targetHost = "10.253.18.102"; require = [ base node ]; }; }