Add nixos and bootstrapping as submodules. Automate apitokens.

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "nixos"]
 	path = nixos
 	url = git@gitlab.itpartner.no:juselius/nixos-configuration.git
-[submodule "kube-system-setup"]
-	path = kube-system-setup
+[submodule "kube-system-bootstrap"]
+	path = kube-system-bootstrap
 	url = git@gitlab.itpartner.no:k8s/kube-system-setup.git

bin/deploy.sh

@@ -1,22 +1,24 @@
 #!/usr/bin/env bash
 
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
+
 if [ $# = 0 ]; then
     echo "usage: deploy.sh name ..."
     exit 1
 fi
 
-if [ ! -f $1/deployment.nix ]; then
+if [ ! -f $DIR/clusters/$1/default.nix ]; then
     echo "error: $1 does not contain a deployment"
     exit 1
 fi
 
-mkdir -p $1/gcroots
+# mkdir -p $1/gcroots
 
-echo "--- Securing certifiates"
-nix-build -o $1/gcroots/certs $1/build.nix
+# echo "--- Securing certifiates"
+# nix-build -o $1/gcroots/certs $1/build.nix
 
 echo "--- Updating deployment"
-nixops modify -d $1 $1/deployment.nix
+nixops modify -d $1 $DIR/clusters/$1
 
 echo "--- Deploying $1"
 nixops deploy -d $* --allow-reboot

bin/initca.sh

@@ -1,3 +1,6 @@
 #!/usr/bin/env bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
 # nix-store --add-root `pwd`/gcroots/initca --indirect -r $(nix-instantiate ./initca.nix)
-nix-build -o gcroots/initca ./initca.nix
+nix-build -o ca $DIR/../lib/initca.nix

clusters/kube1/default.nix

@@ -14,8 +14,16 @@ let
       10.253.18.100 k0-0
       10.253.18.100 gitlab.itpartner.no registry.itpartner.no minio.itpartner.no
     '';
+    adminAuthorizedKeys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDCGrS7PzjPhVnHftYRw7iCD5K1UXnxtFMS0zVLcGH3u daniel.stien@itpartner.no"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDUqiMnQAj5ZkFLjtBLGkVJy2/uH/dnql/92BUMs8a/W7QULnocC2y08dlD+gITP+iKUFYasrYvqBgzKWvxsJmEodkMNr7iBUlKdjiVZNWEM38IgoZbd0iDYIUyDyIlGWRfshc00FX3ecplmylZcDXqFKtGSAafQAt8wZdNmzaHiC0hBYz3x9i2x0lWcq7UXXdNd581BMNj1FqObIoKetKy+4MxZP3oc061HjIxx9m5D6krsWjz+tgkTgjrJGaMKz8aOiLYDw4F9iQSAISeVBSGNU9laPAmbi4t8tcgvBYZVo76GuhLMxRGj6Om2vOJDvbX2mYrSAJ8g5279gbC7mJddEczCyiewt5bRYVzajC8k46bAsxMMkXOVT6YnCz/0X0d8FGlA96NPn2W9oohD2Jx0fVPNJ055AcxU+WYWe5WvCYUAePaUJW/EZSPPY08di4yoJzaJASXCrXtd7aZCh2ndxbZrn3m2KAbjuoBo69CenQGkBM+HjefMFnr9QCiqz2UbrotdQCzPUG1nwhqN409vg7VYQdWuN7wtFBVK7geG/dAJZBbxngNCdcCC4fQUuXV/DjQqOkCkItCYyTRHUHX/Qrdsfm6wrJfcZy5CZQkz9H2/HuMwG7jaiACI+5nAz0A7S6eKnlkoSM9sAOVsP6S4m9eLwbK6GfM4hoeCjNisQ== dag.brattli@itpartner.no"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMWEYtm1u3HiK4q4J5su6iKWfFjLXt9CIlm9Z9BfJYVj jens@itpartner@Jens-HP3"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
+    ];
   };
-  cluster = callPackage ./k8s.nix { inherit settings; };
+  k8s = import ../../lib/k8s.nix ./.;
+  cluster = callPackage k8s { inherit settings; };
 in
 {
   # k1-0 = cluster.host "10.253.18.109" "k1-0";

clusters/kube1/deploy.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-id=kube1
-
-# if [ $# = 0 ]; then
-#     echo "usage: deploy.sh name ..."
-#     exit 1
-# fi
-
-if [ ! -f ./deployment.nix ]; then
-    echo "error: ./ does not contain a deployment"
-    exit 1
-fi
-
-# mkdir -p $1/gcroots
-
-# echo "--- Securing certifiates"
-# nix-build -o $1/gcroots/certs $1/build.nix
-
-echo "--- Updating deployment"
-nixops modify -d $id ./deployment.nix
-
-echo "--- Deploying $id"
-nixops deploy -d $id --allow-reboot
-

lib/k8s.nix

@@ -1,26 +1,63 @@
-{ pkgs, lib, settings, ...}:
+here: { pkgs, lib, settings, ...}:
 with lib;
 let
   cluster-ca = pkgs.stdenv.mkDerivation {
-      name = "cluster-ca";
-      src = ./ca;
-      buildCommand = ''
-        mkdir -p $out
-        cp $src/* $out
-      '';
+    name = "cluster-ca";
+    src = here + /ca;
+    buildCommand = ''
+     mkdir -p $out
+     cp $src/* $out
+    '';
   };
+
+  cfssl-apitoken = pkgs.stdenv.mkDerivation {
+    name = "cfssl-apitoken";
+    buildCommand = ''
+      head -c ${toString (32 / 2)} /dev/urandom | \
+      od -An -t x | tr -d ' ' > $out
+      chmod 400 $out
+    '';
+  };
+
   nixos-kubernetes-join-nodes = workers:
     let
       wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
     in
-    pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
-      #!/bin/sh
-      set -e
-      token=$(cat /var/lib/cfssl/apitoken.secret)
-      for i in ${wrk}; do
-         ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
-      done
+      pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
+        #!/bin/sh
+        set -e
+        token=$(cat /var/lib/cfssl/apitoken.secret)
+        for i in ${wrk}; do
+           ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
+        done
+      '';
+
+  kube-system-bootstrap = pkgs.stdenv.mkDerivation {
+    name = "kube-system-bootstrap";
+    src = ../kube-system-bootstrap;
+    buildCommand = ''
+     mkdir -p $out/bin
+     mkdir -p $out/share/kube-system-bootstrap
+     cp -r $src/* $out/share/kube-system-bootstrap/
+     cd $out/bin
+     ln -s ../share/kube-system-bootstrap/bin/* .
+     ln -s ../share/kube-system-bootstrap/kube-system-bootstrap .
     '';
+  };
+
+  install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" ''
+    #!${pkgs.bash}/bin/bash
+    set -e
+    if [ -d /var/lib/cfssl ]; then
+      cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
+      chown cfssl /var/lib/cfssl/apitoken.secret
+      chmod 600 /var/lib/cfssl/apitoken.secret
+    fi
+    cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
+    chown root /var/lib/kubernetes/secrets/apitoken.secret
+    chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
+  '';
+
   cidr = "10.10.0.0/16";
 in
 rec {
@@ -34,6 +71,7 @@ rec {
       clusterCidr = cidr;
       kubelet.unschedulable = false;
       pki.genCfsslCACert = false;
+      pki.genCfsslAPIToken = false;
       pki.caCertPathPrefix = "${cluster-ca}/ca";
       apiserver = {
         advertiseAddress = settings.masterAddress;
@@ -57,6 +95,7 @@ rec {
     environment.systemPackages = [
       pkgs.kubernetes-helm
       (nixos-kubernetes-join-nodes settings.workers)
+      kube-system-bootstrap
     ];
   };
 
@@ -81,8 +120,8 @@ rec {
 
   baseNixos = name: {
     imports = [
-      (../nixos/hardware-configuration + "/${name}.nix")
       ../nixos/configuration.nix
+      (here + "/${name}.nix")
     ];
     security.pki.certificateFiles = [
       "${cluster-ca}/ca.pem"
@@ -106,6 +145,19 @@ rec {
       firewall.allowedTCPPorts = [ 80 443 111 ];
       firewall.allowedUDPPorts = [ 111 24007 24008 ];
     };
+    users.extraUsers.admin.openssh.authorizedKeys.keys =
+     settings.adminAuthorizedKeys;
+
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "cfssl.service" ];
+      before = [ "cfssl.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "${install-apitoken}";
+      };
+    };
   };
 
   apiserver = ip: name: self: