Changes for kubernets 1.9.

bin/teardown.sh

@@ -12,10 +12,10 @@ f=.$d.$$
 # nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
 # sleep 60
 
-sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f
+# sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f
-nixops modify -d $d $f
+# nixops modify -d $d $f
-nixops deploy -d $d
+# nixops deploy -d $d
-rm $f
+# rm $f
 
 nixops reboot -d $d
 nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"

lib/k8s.nix

@@ -1,7 +1,7 @@
-{ pkgs, kubeMaster, etcdNodes, clusterHosts, certs, ...}:
+{ pkgs, masterNode, etcdNodes, clusterHosts, certs, ...}:
 let
-  kubeApiserver = "https://${kubeMaster}:443";
+  kubeApiserver = "https://${masterNode}:8443";
-  localApiserver = "https://127.0.0.1:8080";
+  localApiserver = "http://127.0.0.1:8080";
   etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
   etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
 in
@@ -55,25 +55,24 @@ rec {
     networking = {
       firewall = {
         enable = true;
-        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
         allowedTCPPorts = [ 53 4194 10250 ];
         allowedUDPPorts = [ 53 ];
         extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
       };
     };
     virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
-    # systemd.services.kube-proxy.path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod];
+    virtualisation.docker.autoPrune.enable = true;
   };
 
   kubeMaster = {
     services.kubernetes = {
       roles = [ "master" ];
-      kubelet.unschedulable = true;
+      kubelet.unschedulable = false;
       apiserver = {
-        address = kubeMaster;
+        address = masterNode;
-        advertiseAddress = kubeMaster;
+        advertiseAddress = masterNode;
         authorizationMode = [ "Node" "RBAC" ];
-        securePort = 443;
+        securePort = 8443;
         tlsKeyFile   = certs.apiserver.key;
         tlsCertFile  = certs.apiserver.cert;
         clientCaFile = certs.ca.cert;
@@ -94,7 +93,7 @@ rec {
       addons.dns.enable = true;
     };
     networking.firewall = {
-      allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
+      allowedTCPPorts = [ 5000 8080 8443 ]; #;4053 ];
       # allowedUDPPorts = [ 4053 ];
     };
     environment.systemPackages = [ pkgs.kubernetes-helm ];
@@ -118,49 +117,71 @@ rec {
     };
   };
 
-  nixosConfig = node: {
+  nixosConfig = instance: {
-    imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ];
+    imports = [
+      (../nixos/hardware-configuration + "/${instance}.nix")
+      ../nixos/configuration.nix
+    ];
+    services.glusterfs = {
+      enable = true;
+      tlsSettings = {
+        caCert = certs.ca.cert;
+        tlsKeyPath = certs.${instance}.key;
+        tlsPem = certs.${instance}.cert;
+      };
+    };
     networking = {
-      hostName = node;
+      hostName = instance;
       extraHosts = clusterHosts;
       firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
-      firewall.allowedTCPPorts = [ 80 443 ];
+      firewall.allowedTCPPorts = [ 80 443 111 ];
+      firewall.allowedUDPPorts = [ 111 24007 24008 ];
     };
     environment.systemPackages = [ pkgs.tshark ];
+    # services.dnsmasq.enable = true;
   };
 
-  worker = host: ip: { config, lib, pkgs, ... }:
+  plain = ip: name: { config, lib, pkgs, ... }:
-    let
-      instance = host;
-      base = nixosConfig host;
-    in
     {
       deployment.targetHost = ip;
-      require = [ base (kubeConfig instance) (kubeNode instance) ];
+      require = [
+        (nixosConfig name)
+      ];
+    };
+
+  worker = ip: name: { config, lib, pkgs, ... }:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (nixosConfig name)
+        (kubeConfig name)
+        (kubeNode name)
+      ];
       services.kubernetes.addons.dns.enable = false;
     };
 
-  server = host: etc: ip: { config, lib, pkgs, ... }:
+  server = ip: name: etc: { config, lib, pkgs, ... }:
-    let
-      instance = host;
-      base = nixosConfig instance;
-      etcd = etcdConfig etc;
-    in
     {
       deployment.targetHost = ip;
-      require = [ base etcd (kubeConfig instance) (kubeNode instance) ];
+      require = [
+        (nixosConfig name)
+        (etcdConfig etc)
+        (kubeConfig name)
+        (kubeNode name)
+      ];
       services.kubernetes.addons.dns.enable = false;
     };
 
-  apiserver = host: ip: etc: { config, lib, pkgs, ... }:
+  apiserver = ip: name: etc: { config, lib, pkgs, ... }:
-    let
-      instance = host;
-      base = nixosConfig instance;
-      etcd = etcdConfig etc;
-    in
     {
       deployment.targetHost = ip;
-      require = [ base etcd (kubeConfig instance) kubeMaster (kubeNode instance) ];
+      require = [
+        (nixosConfig name)
+        (etcdConfig etc)
+        kubeMaster
+        (kubeConfig name)
+        (kubeNode name)
+      ];
       services.dockerRegistry = {
         enable = true;
         listenAddress = "0.0.0.0";

lib/pki.nix

@@ -101,6 +101,15 @@
     };
   };
 
+  trust = name: hosts: gencert rec {
+    inherit name;
+    csr = gencsr {
+      inherit name hosts;
+      cn = name;
+      o = name;
+    };
+  };
+
   kube-proxy = gencert rec {
     name = "kube-proxy";
     csr = gencsr {