Global token, key and cert provisioning works

bin/teardown.sh

@@ -1,22 +1,31 @@
 #!/usr/bin/env bash
 
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
+
 if [ $# != 1 ]; then
     echo "usage: teardown.sh name"
     exit 1
 fi
 
 d=$1
-f=.$d.$$
+f=$DIR/clusters/$d/.$d.$$
 
 # nixops ssh -d $d ${d}0-0 kubectl delete --all pods
 # nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
 # sleep 60
+teardown () {
+    sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $DIR/clusters/$d/default.nix > $f
+    nixops modify -d $d $f
+    nixops deploy -d $d
+    # nixops reboot -d $d
+    # nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
+    rm $f
+}
 
-# sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f
+echo "Are you sure you want to tear down $d? (yes/no)"
-# nixops modify -d $d $f
+read a
-# nixops deploy -d $d
+case $a in
-# rm $f
+    yes) teardown ;;
-
+    *) : ;;
-nixops reboot -d $d
+esac
-nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
 

clusters/kube1/default.nix

@@ -22,17 +22,10 @@ let
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
     ];
   };
-  k8s = import ../../lib/k8s.nix ./.;
+  cluster = callPackage ../../lib/k8s.nix { here = ./.; inherit settings; };
-  cluster = callPackage k8s { inherit settings; };
 in
 {
-  # k1-0 = cluster.host "10.253.18.109" "k1-0";
+  k1-0 = cluster.apiserver "10.253.18.109" "k1-0";
-  # k1-1 = cluster.host "10.253.18.110" "k1-1";
-  # k1-2 = cluster.host "10.253.18.111" "k1-2";
-  k1-0 = self:
-  {
-    require = [ (cluster.apiserver "10.253.18.109" "k1-0") ];
-  };
   k1-1 = cluster.worker "10.253.18.110" "k1-1";
   k1-2 = cluster.worker "10.253.18.111" "k1-2";
 }

lib/k8s.nix

@@ -1,4 +1,4 @@
-here: { pkgs, lib, settings, ...}:
+{ pkgs, lib, settings, here, ...}:
 with lib;
 let
   cluster-ca = pkgs.stdenv.mkDerivation {
@@ -19,18 +19,18 @@ let
     '';
   };
 
-  nixos-kubernetes-join-nodes = workers:
+  #nixos-kubernetes-join-nodes = workers:
-    let
+  #  let
-      wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
+  #    wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
-    in
+  #  in
-      pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
+  #    pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
-        #!/bin/sh
+  #      #!/bin/sh
-        set -e
+  #      set -e
-        token=$(cat /var/lib/cfssl/apitoken.secret)
+  #      token=$(cat /var/lib/cfssl/apitoken.secret)
-        for i in ${wrk}; do
+  #      for i in ${wrk}; do
-           ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
+  #         ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
-        done
+  #      done
-      '';
+  #    '';
 
   kube-system-bootstrap = pkgs.stdenv.mkDerivation {
     name = "kube-system-bootstrap";
@@ -45,18 +45,20 @@ let
     '';
   };
 
-  install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" ''
+  install-apitoken = ''
-    #!${pkgs.bash}/bin/bash
+      #!${pkgs.bash}/bin/bash
-    set -e
+      set -e
-    if [ -d /var/lib/cfssl ]; then
+      if [ -d /var/lib/cfssl ]; then
-      cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
+        cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
-      chown cfssl /var/lib/cfssl/apitoken.secret
+        chown cfssl /var/lib/cfssl/apitoken.secret
-      chmod 600 /var/lib/cfssl/apitoken.secret
+        chmod 640 /var/lib/cfssl/apitoken.secret
-    fi
+      else
-    cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
+        mkdir -p /var/lib/kubernetes/secrets
-    chown root /var/lib/kubernetes/secrets/apitoken.secret
+        cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
-    chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
+        chown root /var/lib/kubernetes/secrets/apitoken.secret
-  '';
+        chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
+      fi
+    '';
 
   cidr = "10.10.0.0/16";
 in
@@ -94,9 +96,19 @@ rec {
     };
     environment.systemPackages = [
       pkgs.kubernetes-helm
-      (nixos-kubernetes-join-nodes settings.workers)
+      # (nixos-kubernetes-join-nodes settings.workers)
       kube-system-bootstrap
     ];
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "cfssl.service" ];
+      before = [ "cfssl.target" ];
+      script = install-apitoken;
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
   };
 
   kubeWorker = {
@@ -116,9 +128,22 @@ rec {
     };
     virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
     virtualisation.docker.autoPrune.enable = true;
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "certmgr.service" ];
+      before = [ "certmgr.target" ];
+      script = install-apitoken;
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
   };
 
   baseNixos = name: {
+    users.extraUsers.admin.openssh.authorizedKeys.keys =
+     settings.adminAuthorizedKeys;
+
     imports = [
       ../nixos/configuration.nix
       (here + "/${name}.nix")
@@ -145,19 +170,6 @@ rec {
       firewall.allowedTCPPorts = [ 80 443 111 ];
       firewall.allowedUDPPorts = [ 111 24007 24008 ];
     };
-    users.extraUsers.admin.openssh.authorizedKeys.keys =
-     settings.adminAuthorizedKeys;
-
-    systemd.services.kube-certmgr-apitoken-bootstrap = {
-      description = "Kubernetes certmgr bootstrapper";
-      wantedBy = [ "cfssl.service" ];
-      before = [ "cfssl.target" ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-        ExecStart = "${install-apitoken}";
-      };
-    };
   };
 
   apiserver = ip: name: self: