diff --git a/bin/teardown.sh b/bin/teardown.sh index 839645e..355cd6e 100755 --- a/bin/teardown.sh +++ b/bin/teardown.sh @@ -12,10 +12,10 @@ f=.$d.$$ # nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods # sleep 60 -sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f -nixops modify -d $d $f -nixops deploy -d $d -rm $f +# sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f +# nixops modify -d $d $f +# nixops deploy -d $d +# rm $f nixops reboot -d $d nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd" diff --git a/lib/k8s.nix b/lib/k8s.nix index 20345bb..2395a2f 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -1,7 +1,7 @@ -{ pkgs, kubeMaster, etcdNodes, clusterHosts, certs, ...}: +{ pkgs, masterNode, etcdNodes, clusterHosts, certs, ...}: let - kubeApiserver = "https://${kubeMaster}:443"; - localApiserver = "https://127.0.0.1:8080"; + kubeApiserver = "https://${masterNode}:8443"; + localApiserver = "http://127.0.0.1:8080"; etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes; etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes; in @@ -55,25 +55,24 @@ rec { networking = { firewall = { enable = true; - # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; allowedTCPPorts = [ 53 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; - # systemd.services.kube-proxy.path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod]; + virtualisation.docker.autoPrune.enable = true; }; kubeMaster = { services.kubernetes = { roles = [ "master" ]; - kubelet.unschedulable = true; + kubelet.unschedulable = false; apiserver = { - address = kubeMaster; - advertiseAddress = kubeMaster; + address = masterNode; + advertiseAddress = masterNode; authorizationMode = [ "Node" "RBAC" ]; - securePort = 443; + securePort = 8443; tlsKeyFile = certs.apiserver.key; tlsCertFile = certs.apiserver.cert; clientCaFile = certs.ca.cert; @@ -94,7 +93,7 @@ rec { addons.dns.enable = true; }; networking.firewall = { - allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ]; + allowedTCPPorts = [ 5000 8080 8443 ]; #;4053 ]; # allowedUDPPorts = [ 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm ]; @@ -118,49 +117,71 @@ rec { }; }; - nixosConfig = node: { - imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ]; + nixosConfig = instance: { + imports = [ + (../nixos/hardware-configuration + "/${instance}.nix") + ../nixos/configuration.nix + ]; + services.glusterfs = { + enable = true; + tlsSettings = { + caCert = certs.ca.cert; + tlsKeyPath = certs.${instance}.key; + tlsPem = certs.${instance}.cert; + }; + }; networking = { - hostName = node; + hostName = instance; extraHosts = clusterHosts; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; - firewall.allowedTCPPorts = [ 80 443 ]; + firewall.allowedTCPPorts = [ 80 443 111 ]; + firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; environment.systemPackages = [ pkgs.tshark ]; + # services.dnsmasq.enable = true; }; - worker = host: ip: { config, lib, pkgs, ... }: - let - instance = host; - base = nixosConfig host; - in + plain = ip: name: { config, lib, pkgs, ... }: { deployment.targetHost = ip; - require = [ base (kubeConfig instance) (kubeNode instance) ]; + require = [ + (nixosConfig name) + ]; + }; + + worker = ip: name: { config, lib, pkgs, ... }: + { + deployment.targetHost = ip; + require = [ + (nixosConfig name) + (kubeConfig name) + (kubeNode name) + ]; services.kubernetes.addons.dns.enable = false; }; - server = host: etc: ip: { config, lib, pkgs, ... }: - let - instance = host; - base = nixosConfig instance; - etcd = etcdConfig etc; - in + server = ip: name: etc: { config, lib, pkgs, ... }: { deployment.targetHost = ip; - require = [ base etcd (kubeConfig instance) (kubeNode instance) ]; + require = [ + (nixosConfig name) + (etcdConfig etc) + (kubeConfig name) + (kubeNode name) + ]; services.kubernetes.addons.dns.enable = false; }; - apiserver = host: ip: etc: { config, lib, pkgs, ... }: - let - instance = host; - base = nixosConfig instance; - etcd = etcdConfig etc; - in + apiserver = ip: name: etc: { config, lib, pkgs, ... }: { deployment.targetHost = ip; - require = [ base etcd (kubeConfig instance) kubeMaster (kubeNode instance) ]; + require = [ + (nixosConfig name) + (etcdConfig etc) + kubeMaster + (kubeConfig name) + (kubeNode name) + ]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; diff --git a/lib/pki.nix b/lib/pki.nix index 793d750..e2f2771 100644 --- a/lib/pki.nix +++ b/lib/pki.nix @@ -101,6 +101,15 @@ }; }; + trust = name: hosts: gencert rec { + inherit name; + csr = gencsr { + inherit name hosts; + cn = name; + o = name; + }; + }; + kube-proxy = gencert rec { name = "kube-proxy"; csr = gencsr {