with import ./base/pki.nix; let server-cert = mkCert { name = "kubernetes"; csr = csr { cn = "kubernetes"; hosts = ''"kubernetes", "k8s0-0", "10.253.18.100"''; }; profile = "server"; }; etcd0-cert = mkCert { name = "etcd0"; csr = csr { cn = "etcd0"; hosts = ''"etcd0", "10.253.18.100"''; }; profile = "peer"; }; etcd1-cert = mkCert { name = "etcd1"; csr = csr { cn = "etcd1"; hosts = ''"etcd1", "10.253.18.101"''; }; profile = "peer"; }; client-cert = mkCert { name = "client"; csr = csr { cn = "client"; hosts = ''''; }; profile = "client"; }; server_key = "${server-cert}/cert-key.pem"; server_cert = "${server-cert}/cert.pem"; etcd0_key = "${etcd0-cert}/cert-key.pem"; etcd0_cert = "${etcd0-cert}/cert.pem"; etcd1_key = "${etcd1-cert}/cert-key.pem"; etcd1_cert = "${etcd1-cert}/cert.pem"; client_key = "${client-cert}/cert-key.pem"; client_cert = "${client-cert}/cert.pem"; etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ]; etcdConfig = etcd: { services.etcd = { enable = true; listenClientUrls = ["https://0.0.0.0:2379"]; listenPeerUrls = ["https://0.0.0.0:2380"]; peerClientCertAuth = true; keyFile = "${etcd.key}"; certFile = "${etcd.cert}"; trustedCaFile = "${ca_cert}"; advertiseClientUrls = [ "https://${etcd.name}:2379" ]; initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ]; initialCluster = etcdCluster; }; environment.variables = { ETCDCTL_KEY_FILE = "${etcd.key}"; ETCDCTL_CERT_FILE = "${etcd.cert}"; ETCDCTL_CA_FILE = "${ca_cert}"; ETCDCTL_PEERS = "https://127.0.0.1:2379"; }; networking.firewall.allowedTCPPorts = [ 2379 2380 ]; }; flannelConfig = { services.flannel = { enable = true; network = "10.10.0.0/16"; iface = "enp2s0"; etcd = { endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ]; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; }; }; kubeConfig = { require = [ flannelConfig ]; networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN networking.firewall.allowedTCPPorts = [ 10250 ]; systemd.services.docker = { after = [ "flannel.service" ]; serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; }; virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; services.kubernetes.etcd = { servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; # services.kubernetes.verbose = true; }; kubeNode = { services.kubernetes = { roles = [ "node" ]; kubeconfig = { server = "https://kubernetes:443"; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; kubelet = { tlsKeyFile = "${client_key}"; tlsCertFile = "${client_cert}"; extraOpts = "--client-ca-file=${ca_cert}"; networkPlugin = null; clusterDns = "kubernetes"; }; }; }; kubeMaster = { services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; }; services.kubernetes = { roles = [ "master" ]; apiserver = { publicAddress = "0.0.0.0"; address = "0.0.0.0"; clientCaFile = "${ca_cert}"; tlsKeyFile = "${server_key}"; tlsCertFile = "${server_cert}"; # kubeletClientCaFile = "${ca_cert}"; # kubeletClientKeyFile = "${server_key}"; # kubeletClientCertFile = "${server_cert}"; }; scheduler.leaderElect = true; controllerManager.leaderElect = true; controllerManager.serviceAccountKeyFile = "${server_key}"; }; networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; networking.firewall.allowedUDPPorts = [ 53 ]; systemd.services.flannel.after = [ "etcd.service" ]; }; baseConfig = node: { imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; require = [ kubeConfig ]; networking.hostName = node; networking.extraHosts = '' 10.253.18.100 etcd0 kubernetes 10.253.18.101 etcd1 ''; }; in { k8s0-0 = { config, lib, pkgs, ... }: let host = "k8s0-0"; base = baseConfig host; etcd = etcdConfig { name = "etcd0"; key = etcd0_key; cert = etcd0_cert; }; in { deployment.targetHost = "10.253.18.100"; require = [ base etcd kubeMaster kubeNode ]; }; k8s0-1 = { config, lib, pkgs, ... }: let host = "k8s0-1"; base = baseConfig host; etcd = etcdConfig { name = "etcd1"; key = etcd1_key; cert = etcd1_cert; }; in { deployment.targetHost = "10.253.18.101"; require = [ base etcd kubeNode ]; }; k8s0-2 = { config, lib, pkgs, ... }: let host = "k8s0-2"; base = baseConfig host; in { deployment.targetHost = "10.253.18.102"; require = [ base kubeNode ]; }; }