with import ./base/pki.nix; let server-cert = mkCert { name = "kubernetes"; csr = csr { cn = "kubernetes"; hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"''; }; profile = "server"; }; etcd0-cert = mkCert { name = "etcd0"; csr = csr { cn = "etcd0"; hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"''; }; profile = "peer"; }; etcd1-cert = mkCert { name = "etcd1"; csr = csr { cn = "etcd1"; hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"''; }; profile = "peer"; }; client-cert = mkCert { name = "client"; csr = csr { cn = "client"; hosts = ''''; }; profile = "client"; }; server_key = "${server-cert}/cert-key.pem"; server_cert = "${server-cert}/cert.pem"; etcd0_key = "${etcd0-cert}/cert-key.pem"; etcd0_cert = "${etcd0-cert}/cert.pem"; etcd1_key = "${etcd1-cert}/cert-key.pem"; etcd1_cert = "${etcd1-cert}/cert.pem"; client_key = "${client-cert}/cert-key.pem"; client_cert = "${client-cert}/cert.pem"; etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ]; etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ]; etcdConfig = etcd: { services.etcd = { name = etcd.name; enable = true; listenClientUrls = ["https://0.0.0.0:2379"]; listenPeerUrls = ["https://0.0.0.0:2380"]; peerClientCertAuth = true; keyFile = "${etcd.key}"; certFile = "${etcd.cert}"; trustedCaFile = "${ca_cert}"; advertiseClientUrls = [ "https://${etcd.name}:2379" ]; initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ]; initialCluster = etcdCluster; }; environment.variables = { ETCDCTL_KEY_FILE = "${etcd.key}"; ETCDCTL_CERT_FILE = "${etcd.cert}"; ETCDCTL_CA_FILE = "${ca_cert}"; ETCDCTL_PEERS = "https://localhost:2379"; }; # networking.firewall.allowedTCPPorts = [ 2379 2380 ]; systemd.services.flannel.after = [ "etcd.service" ]; }; flannelConfig = { services.flannel = { enable = true; network = "10.10.0.0/16"; iface = "enp2s0"; etcd = { endpoints = etcdServers; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; }; # networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN }; kubeConfig = { systemd.services.docker = { after = [ "flannel.service" ]; serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; }; virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; services.kubernetes.etcd = { servers = etcdServers; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; # services.kubernetes.verbose = true; }; kubeNode = { services.kubernetes = { roles = [ "node" ]; kubeconfig = { server = "https://10.253.18.100:443"; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; kubelet = { tlsKeyFile = "${client_key}"; tlsCertFile = "${client_cert}"; networkPlugin = null; clusterDns = "10.10.21.0"; }; }; networking.firewall = { enable = false; # allowedTCPPorts = [ 53 10250 8000 8080 ]; # allowedUDPPorts = [ 53 ]; # trustedInterfaces = [ "flannel.1" "docker0" ]; # extraCommands = '' # iptables -P FORWARD ACCEPT # ''; }; }; kubeMaster = { services.kubernetes = { roles = [ "master" ]; apiserver = { publicAddress = "0.0.0.0"; address = "0.0.0.0"; clientCaFile = "${ca_cert}"; tlsKeyFile = "${server_key}"; tlsCertFile = "${server_cert}"; # serviceAccountKeyFile = "${server_key}"; # kubeletClientCaFile = "${ca_cert}"; # kubeletClientKeyFile = "${client_key}"; # kubeletClientCertFile = "${client_cert}"; }; scheduler.leaderElect = true; controllerManager.leaderElect = true; controllerManager.serviceAccountKeyFile = "${server_key}"; }; # networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; # networking.firewall.allowedUDPPorts = [ 53 ]; }; baseConfig = node: { imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; networking.hostName = node; networking.extraHosts = '' 10.253.18.100 etcd0 k8s0-0 kubernetes 10.253.18.101 etcd1 ''; virtualisation.docker.enable = true; }; etcdConf0 = etcdConfig { name = "etcd0"; key = etcd0_key; cert = etcd0_cert; }; etcdConf1 = etcdConfig { name = "etcd1"; key = etcd1_key; cert = etcd1_cert; }; minion = host: ip: { config, lib, pkgs, ... }: let inherit host; base = baseConfig host; in { deployment.targetHost = ip; require = [ base flannelConfig kubeConfig kubeNode ]; }; in { k8s0-0 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-0"; etcd = etcdConf0; in { deployment.targetHost = "10.253.18.100"; require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; }; }; k8s0-1 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-1"; etcd = etcdConf1; in { deployment.targetHost = "10.253.18.101"; require = [ base etcd flannelConfig kubeConfig kubeNode ]; }; k8s0-2 = minion "k8s0-2" "10.253.18.102"; }