From 286ad04f8ebd73a5b0dfa382ee43ce38b8adb277 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Wed, 16 Oct 2019 15:02:21 +0200 Subject: [PATCH] Add nixos and bootstrapping as submodules. Automate apitokens. --- .gitmodules | 4 +- bin/deploy.sh | 12 +++--- bin/initca.sh | 5 ++- clusters/kube1/default.nix | 10 ++++- clusters/kube1/deploy.sh | 25 ------------ kube-system-bootstrap | 1 + kube-system-setup | 1 - lib/k8s.nix | 82 +++++++++++++++++++++++++++++++------- nixos | 2 +- 9 files changed, 91 insertions(+), 51 deletions(-) delete mode 100755 clusters/kube1/deploy.sh create mode 160000 kube-system-bootstrap delete mode 160000 kube-system-setup diff --git a/.gitmodules b/.gitmodules index f0eccdb..5ba417e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "nixos"] path = nixos url = git@gitlab.itpartner.no:juselius/nixos-configuration.git -[submodule "kube-system-setup"] - path = kube-system-setup +[submodule "kube-system-bootstrap"] + path = kube-system-bootstrap url = git@gitlab.itpartner.no:k8s/kube-system-setup.git diff --git a/bin/deploy.sh b/bin/deploy.sh index c39508e..15005e9 100755 --- a/bin/deploy.sh +++ b/bin/deploy.sh @@ -1,22 +1,24 @@ #!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + if [ $# = 0 ]; then echo "usage: deploy.sh name ..." exit 1 fi -if [ ! -f $1/deployment.nix ]; then +if [ ! -f $DIR/clusters/$1/default.nix ]; then echo "error: $1 does not contain a deployment" exit 1 fi -mkdir -p $1/gcroots +# mkdir -p $1/gcroots -echo "--- Securing certifiates" -nix-build -o $1/gcroots/certs $1/build.nix +# echo "--- Securing certifiates" +# nix-build -o $1/gcroots/certs $1/build.nix echo "--- Updating deployment" -nixops modify -d $1 $1/deployment.nix +nixops modify -d $1 $DIR/clusters/$1 echo "--- Deploying $1" nixops deploy -d $* --allow-reboot diff --git a/bin/initca.sh b/bin/initca.sh index b37ccbe..1555d00 100755 --- a/bin/initca.sh +++ b/bin/initca.sh @@ -1,3 +1,6 @@ #!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # nix-store --add-root `pwd`/gcroots/initca --indirect -r $(nix-instantiate ./initca.nix) -nix-build -o gcroots/initca ./initca.nix +nix-build -o ca $DIR/../lib/initca.nix diff --git a/clusters/kube1/default.nix b/clusters/kube1/default.nix index 6d93704..b370acd 100644 --- a/clusters/kube1/default.nix +++ b/clusters/kube1/default.nix @@ -14,8 +14,16 @@ let 10.253.18.100 k0-0 10.253.18.100 gitlab.itpartner.no registry.itpartner.no minio.itpartner.no ''; + adminAuthorizedKeys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDCGrS7PzjPhVnHftYRw7iCD5K1UXnxtFMS0zVLcGH3u daniel.stien@itpartner.no" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDUqiMnQAj5ZkFLjtBLGkVJy2/uH/dnql/92BUMs8a/W7QULnocC2y08dlD+gITP+iKUFYasrYvqBgzKWvxsJmEodkMNr7iBUlKdjiVZNWEM38IgoZbd0iDYIUyDyIlGWRfshc00FX3ecplmylZcDXqFKtGSAafQAt8wZdNmzaHiC0hBYz3x9i2x0lWcq7UXXdNd581BMNj1FqObIoKetKy+4MxZP3oc061HjIxx9m5D6krsWjz+tgkTgjrJGaMKz8aOiLYDw4F9iQSAISeVBSGNU9laPAmbi4t8tcgvBYZVo76GuhLMxRGj6Om2vOJDvbX2mYrSAJ8g5279gbC7mJddEczCyiewt5bRYVzajC8k46bAsxMMkXOVT6YnCz/0X0d8FGlA96NPn2W9oohD2Jx0fVPNJ055AcxU+WYWe5WvCYUAePaUJW/EZSPPY08di4yoJzaJASXCrXtd7aZCh2ndxbZrn3m2KAbjuoBo69CenQGkBM+HjefMFnr9QCiqz2UbrotdQCzPUG1nwhqN409vg7VYQdWuN7wtFBVK7geG/dAJZBbxngNCdcCC4fQUuXV/DjQqOkCkItCYyTRHUHX/Qrdsfm6wrJfcZy5CZQkz9H2/HuMwG7jaiACI+5nAz0A7S6eKnlkoSM9sAOVsP6S4m9eLwbK6GfM4hoeCjNisQ== dag.brattli@itpartner.no" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMWEYtm1u3HiK4q4J5su6iKWfFjLXt9CIlm9Z9BfJYVj jens@itpartner@Jens-HP3" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no" + ]; }; - cluster = callPackage ./k8s.nix { inherit settings; }; + k8s = import ../../lib/k8s.nix ./.; + cluster = callPackage k8s { inherit settings; }; in { # k1-0 = cluster.host "10.253.18.109" "k1-0"; diff --git a/clusters/kube1/deploy.sh b/clusters/kube1/deploy.sh deleted file mode 100755 index c2b6210..0000000 --- a/clusters/kube1/deploy.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -id=kube1 - -# if [ $# = 0 ]; then -# echo "usage: deploy.sh name ..." -# exit 1 -# fi - -if [ ! -f ./deployment.nix ]; then - echo "error: ./ does not contain a deployment" - exit 1 -fi - -# mkdir -p $1/gcroots - -# echo "--- Securing certifiates" -# nix-build -o $1/gcroots/certs $1/build.nix - -echo "--- Updating deployment" -nixops modify -d $id ./deployment.nix - -echo "--- Deploying $id" -nixops deploy -d $id --allow-reboot - diff --git a/kube-system-bootstrap b/kube-system-bootstrap new file mode 160000 index 0000000..a0572ff --- /dev/null +++ b/kube-system-bootstrap @@ -0,0 +1 @@ +Subproject commit a0572ff7a77a1e8057b4dc6230bd1e69c00cc307 diff --git a/kube-system-setup b/kube-system-setup deleted file mode 160000 index c5cff09..0000000 --- a/kube-system-setup +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c5cff093cd3b2c9aa0356cda31bb5f06f430db0e diff --git a/lib/k8s.nix b/lib/k8s.nix index accdb62..806a7bb 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -1,26 +1,63 @@ -{ pkgs, lib, settings, ...}: +here: { pkgs, lib, settings, ...}: with lib; let cluster-ca = pkgs.stdenv.mkDerivation { - name = "cluster-ca"; - src = ./ca; - buildCommand = '' - mkdir -p $out - cp $src/* $out - ''; + name = "cluster-ca"; + src = here + /ca; + buildCommand = '' + mkdir -p $out + cp $src/* $out + ''; }; + + cfssl-apitoken = pkgs.stdenv.mkDerivation { + name = "cfssl-apitoken"; + buildCommand = '' + head -c ${toString (32 / 2)} /dev/urandom | \ + od -An -t x | tr -d ' ' > $out + chmod 400 $out + ''; + }; + nixos-kubernetes-join-nodes = workers: let wrk = builtins.foldl' (a: s: a + " " + s) "" workers; in - pkgs.writeScriptBin "nixos-kubernetes-join-nodes" '' - #!/bin/sh - set -e - token=$(cat /var/lib/cfssl/apitoken.secret) - for i in ${wrk}; do - ssh root@$i "echo $token | sh nixos-kubernetes-node-join" - done + pkgs.writeScriptBin "nixos-kubernetes-join-nodes" '' + #!/bin/sh + set -e + token=$(cat /var/lib/cfssl/apitoken.secret) + for i in ${wrk}; do + ssh root@$i "echo $token | sh nixos-kubernetes-node-join" + done + ''; + + kube-system-bootstrap = pkgs.stdenv.mkDerivation { + name = "kube-system-bootstrap"; + src = ../kube-system-bootstrap; + buildCommand = '' + mkdir -p $out/bin + mkdir -p $out/share/kube-system-bootstrap + cp -r $src/* $out/share/kube-system-bootstrap/ + cd $out/bin + ln -s ../share/kube-system-bootstrap/bin/* . + ln -s ../share/kube-system-bootstrap/kube-system-bootstrap . ''; + }; + + install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" '' + #!${pkgs.bash}/bin/bash + set -e + if [ -d /var/lib/cfssl ]; then + cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret + chown cfssl /var/lib/cfssl/apitoken.secret + chmod 600 /var/lib/cfssl/apitoken.secret + fi + cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret + chown root /var/lib/kubernetes/secrets/apitoken.secret + chmod 600 /var/lib/kubernetes/secrets/apitoken.secret + ''; + cidr = "10.10.0.0/16"; in rec { @@ -34,6 +71,7 @@ rec { clusterCidr = cidr; kubelet.unschedulable = false; pki.genCfsslCACert = false; + pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${cluster-ca}/ca"; apiserver = { advertiseAddress = settings.masterAddress; @@ -57,6 +95,7 @@ rec { environment.systemPackages = [ pkgs.kubernetes-helm (nixos-kubernetes-join-nodes settings.workers) + kube-system-bootstrap ]; }; @@ -81,8 +120,8 @@ rec { baseNixos = name: { imports = [ - (../nixos/hardware-configuration + "/${name}.nix") ../nixos/configuration.nix + (here + "/${name}.nix") ]; security.pki.certificateFiles = [ "${cluster-ca}/ca.pem" @@ -106,6 +145,19 @@ rec { firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; + users.extraUsers.admin.openssh.authorizedKeys.keys = + settings.adminAuthorizedKeys; + + systemd.services.kube-certmgr-apitoken-bootstrap = { + description = "Kubernetes certmgr bootstrapper"; + wantedBy = [ "cfssl.service" ]; + before = [ "cfssl.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${install-apitoken}"; + }; + }; }; apiserver = ip: name: self: diff --git a/nixos b/nixos index 82d6017..5fb88d7 160000 --- a/nixos +++ b/nixos @@ -1 +1 @@ -Subproject commit 82d60179d2962ca0503d5e1707557c17158cf07f +Subproject commit 5fb88d7ab6eb2236007aeeeab3eefa8ba1c39f36