From 2dae12bad2d6ef7953d1bcc11e222acb5190e57c Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Wed, 16 Oct 2019 19:14:37 +0200 Subject: [PATCH] Global token, key and cert provisioning works --- bin/teardown.sh | 25 +++++++---- clusters/kube1/default.nix | 11 +---- lib/k8s.nix | 90 +++++++++++++++++++++----------------- 3 files changed, 70 insertions(+), 56 deletions(-) diff --git a/bin/teardown.sh b/bin/teardown.sh index 3d39758..e501cf2 100755 --- a/bin/teardown.sh +++ b/bin/teardown.sh @@ -1,22 +1,31 @@ #!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + if [ $# != 1 ]; then echo "usage: teardown.sh name" exit 1 fi d=$1 -f=.$d.$$ +f=$DIR/clusters/$d/.$d.$$ # nixops ssh -d $d ${d}0-0 kubectl delete --all pods # nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods # sleep 60 +teardown () { + sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $DIR/clusters/$d/default.nix > $f + nixops modify -d $d $f + nixops deploy -d $d + # nixops reboot -d $d + # nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl" + rm $f +} -# sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f -# nixops modify -d $d $f -# nixops deploy -d $d -# rm $f - -nixops reboot -d $d -nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl" +echo "Are you sure you want to tear down $d? (yes/no)" +read a +case $a in + yes) teardown ;; + *) : ;; +esac diff --git a/clusters/kube1/default.nix b/clusters/kube1/default.nix index b370acd..a7a6e3e 100644 --- a/clusters/kube1/default.nix +++ b/clusters/kube1/default.nix @@ -22,17 +22,10 @@ let "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no" ]; }; - k8s = import ../../lib/k8s.nix ./.; - cluster = callPackage k8s { inherit settings; }; + cluster = callPackage ../../lib/k8s.nix { here = ./.; inherit settings; }; in { - # k1-0 = cluster.host "10.253.18.109" "k1-0"; - # k1-1 = cluster.host "10.253.18.110" "k1-1"; - # k1-2 = cluster.host "10.253.18.111" "k1-2"; - k1-0 = self: - { - require = [ (cluster.apiserver "10.253.18.109" "k1-0") ]; - }; + k1-0 = cluster.apiserver "10.253.18.109" "k1-0"; k1-1 = cluster.worker "10.253.18.110" "k1-1"; k1-2 = cluster.worker "10.253.18.111" "k1-2"; } diff --git a/lib/k8s.nix b/lib/k8s.nix index 806a7bb..b6746f1 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -1,4 +1,4 @@ -here: { pkgs, lib, settings, ...}: +{ pkgs, lib, settings, here, ...}: with lib; let cluster-ca = pkgs.stdenv.mkDerivation { @@ -19,18 +19,18 @@ let ''; }; - nixos-kubernetes-join-nodes = workers: - let - wrk = builtins.foldl' (a: s: a + " " + s) "" workers; - in - pkgs.writeScriptBin "nixos-kubernetes-join-nodes" '' - #!/bin/sh - set -e - token=$(cat /var/lib/cfssl/apitoken.secret) - for i in ${wrk}; do - ssh root@$i "echo $token | sh nixos-kubernetes-node-join" - done - ''; + #nixos-kubernetes-join-nodes = workers: + # let + # wrk = builtins.foldl' (a: s: a + " " + s) "" workers; + # in + # pkgs.writeScriptBin "nixos-kubernetes-join-nodes" '' + # #!/bin/sh + # set -e + # token=$(cat /var/lib/cfssl/apitoken.secret) + # for i in ${wrk}; do + # ssh root@$i "echo $token | sh nixos-kubernetes-node-join" + # done + # ''; kube-system-bootstrap = pkgs.stdenv.mkDerivation { name = "kube-system-bootstrap"; @@ -45,18 +45,20 @@ let ''; }; - install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" '' - #!${pkgs.bash}/bin/bash - set -e - if [ -d /var/lib/cfssl ]; then - cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret - chown cfssl /var/lib/cfssl/apitoken.secret - chmod 600 /var/lib/cfssl/apitoken.secret - fi - cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret - chown root /var/lib/kubernetes/secrets/apitoken.secret - chmod 600 /var/lib/kubernetes/secrets/apitoken.secret - ''; + install-apitoken = '' + #!${pkgs.bash}/bin/bash + set -e + if [ -d /var/lib/cfssl ]; then + cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret + chown cfssl /var/lib/cfssl/apitoken.secret + chmod 640 /var/lib/cfssl/apitoken.secret + else + mkdir -p /var/lib/kubernetes/secrets + cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret + chown root /var/lib/kubernetes/secrets/apitoken.secret + chmod 600 /var/lib/kubernetes/secrets/apitoken.secret + fi + ''; cidr = "10.10.0.0/16"; in @@ -94,9 +96,19 @@ rec { }; environment.systemPackages = [ pkgs.kubernetes-helm - (nixos-kubernetes-join-nodes settings.workers) + # (nixos-kubernetes-join-nodes settings.workers) kube-system-bootstrap ]; + systemd.services.kube-certmgr-apitoken-bootstrap = { + description = "Kubernetes certmgr bootstrapper"; + wantedBy = [ "cfssl.service" ]; + before = [ "cfssl.target" ]; + script = install-apitoken; + serviceConfig = { + RestartSec = "10s"; + Restart = "on-failure"; + }; + }; }; kubeWorker = { @@ -116,9 +128,22 @@ rec { }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; + systemd.services.kube-certmgr-apitoken-bootstrap = { + description = "Kubernetes certmgr bootstrapper"; + wantedBy = [ "certmgr.service" ]; + before = [ "certmgr.target" ]; + script = install-apitoken; + serviceConfig = { + RestartSec = "10s"; + Restart = "on-failure"; + }; + }; }; baseNixos = name: { + users.extraUsers.admin.openssh.authorizedKeys.keys = + settings.adminAuthorizedKeys; + imports = [ ../nixos/configuration.nix (here + "/${name}.nix") @@ -145,19 +170,6 @@ rec { firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; - users.extraUsers.admin.openssh.authorizedKeys.keys = - settings.adminAuthorizedKeys; - - systemd.services.kube-certmgr-apitoken-bootstrap = { - description = "Kubernetes certmgr bootstrapper"; - wantedBy = [ "cfssl.service" ]; - before = [ "cfssl.target" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = "${install-apitoken}"; - }; - }; }; apiserver = ip: name: self: