with import ./certs.nix; let pkgs = import {}; kube_apiserver = "https://10.253.18.100:443"; etcdServers = [ "etcd0" "etcd1" "etcd2" ]; etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers; etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers; etcdConfig = name: { services.etcd = { inherit name; enable = true; listenClientUrls = ["https://0.0.0.0:2379"]; listenPeerUrls = ["https://0.0.0.0:2380"]; peerClientCertAuth = true; keyFile = etcd_key; certFile = etcd_cert; trustedCaFile = ca_pem; advertiseClientUrls = [ "https://${name}:2379" ]; initialAdvertisePeerUrls = [ "https://${name}:2380" ]; initialCluster = etcdCluster; }; environment.variables = { ETCDCTL_KEY_FILE = "${etcd_client_key}"; ETCDCTL_CERT_FILE = "${etcd_client_cert}"; ETCDCTL_CA_FILE = "${ca_pem}"; ETCDCTL_PEERS = "https://127.0.0.1:2379"; }; networking.firewall.allowedTCPPorts = [ 2379 2380 ]; systemd.services.flannel.after = [ "etcd.service" ]; }; kubeconfig = { caFile = ca_pem; keyFile = worker_key; certFile = worker_cert; server = kube_apiserver; }; kubeNode = { services.kubernetes = { roles = [ "node" ]; kubeconfig = { server = kube_apiserver; keyFile = worker_key; certFile = worker_cert; caFile = ca_pem; }; kubelet = { enable = true; clientCaFile = ca_pem; tlsKeyFile = worker_key; tlsCertFile = worker_cert; networkPlugin = null; # clusterDns = "10.253.18.100"; clusterDns = "10.0.0.254"; inherit kubeconfig; }; }; networking = { firewall = { enable = true; # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; allowedTCPPorts = [ 53 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; }; kubeMaster = { services.kubernetes = { roles = [ "master" ]; kubelet.unschedulable = true; apiserver = { address = "0.0.0.0"; publicAddress = "0.0.0.0"; advertiseAddress = "10.253.18.100"; securePort = 443; tlsKeyFile = apiserver_key; tlsCertFile = apiserver_cert; clientCaFile = ca_pem; kubeletClientCaFile = ca_pem; kubeletClientKeyFile = worker_key; kubeletClientCertFile = worker_cert; serviceAccountKeyFile = apiserver_key; }; scheduler.leaderElect = true; controllerManager = { leaderElect = true; serviceAccountKeyFile = apiserver_key; rootCaFile = ca_pem; inherit kubeconfig; }; addons.dashboard.enable = true; addons.dns.enable = true; }; networking.firewall = { allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ]; # allowedUDPPorts = [ 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm ]; }; kubeConfig = { services.kubernetes = { verbose = false; caFile = ca_pem; flannel.enable = true; clusterCidr = "10.10.0.0/16"; etcd = { servers = etcdEndpoints; keyFile = etcd_client_key; certFile = etcd_client_cert; caFile = ca_pem; }; proxy = { inherit kubeconfig; }; }; }; baseConfig = node: { imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; networking = { hostName = node; extraHosts = '' 10.253.18.100 etcd0 kubernetes 10.253.18.101 etcd1 10.253.18.102 etcd2 ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 ]; }; services.dnsmasq.enable = true; services.dnsmasq.servers = [ "/cluster.local/10.0.0.254#53" ]; }; minion = host: ip: { config, lib, pkgs, ... }: let inherit host; base = baseConfig host; in { deployment.targetHost = ip; require = [ base kubeConfig kubeNode ]; services.kubernetes.addons.dns.enable = false; }; in { k8s0-0 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-0"; etcd = etcdConfig "etcd0"; in { deployment.targetHost = "10.253.18.100"; require = [ base etcd kubeConfig kubeMaster kubeNode ]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; extraConfig = { REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver_cert}"; REGISTRY_HTTP_TLS_KEY = "${apiserver_key}"; }; }; }; k8s0-1 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-1"; etcd = etcdConfig "etcd1"; in { deployment.targetHost = "10.253.18.101"; require = [ base etcd kubeConfig kubeNode ]; services.kubernetes.addons.dns.enable = false; }; k8s0-2 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-2"; etcd = etcdConfig "etcd2"; in { deployment.targetHost = "10.253.18.102"; require = [ base etcd kubeConfig kubeNode ]; services.kubernetes.addons.dns.enable = false; }; git01 = { config, lib, pkgs, ... }: let base = baseConfig "git01"; in { deployment.targetHost = "10.253.18.103"; require = [ base kubeConfig kubeNode ]; services.kubernetes.addons.dns.enable = false; services.nfs.server = { enable=true; exports= '' /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) ''; }; networking.firewall.allowedTCPPorts = [ 111 2049 ]; networking.firewall.allowedUDPPorts = [ 111 2049 ]; }; }