oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Global token, key and cert provisioning works

Browse Source
This commit is contained in:
Jonas Juselius
2019-10-16 19:14:37 +02:00
parent 286ad04f8e
commit 2dae12bad2
3 changed files with 70 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
bin/teardown.sh
View File

@@ -1,22 +1,31 @@
#!/usr/bin/env bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
if [ $# != 1 ]; then
echo "usage: teardown.sh name"
exit 1
fi
d=$1
f=.$d.$$
f=$DIR/clusters/$d/.$d.$$
# nixops ssh -d $d ${d}0-0 kubectl delete --all pods
# nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
# sleep 60
teardown () {
sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $DIR/clusters/$d/default.nix > $f
nixops modify -d $d $f
nixops deploy -d $d
# nixops reboot -d $d
# nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
rm $f
}
# sed -s 's/require = \[ \+base .*/require = [ base ];/' $d.nix > $f
# nixops modify -d $d $f
# nixops deploy -d $d
# rm $f
nixops reboot -d $d
nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
echo "Are you sure you want to tear down $d? (yes/no)"
read a
case $a in
yes) teardown ;;
*) : ;;
esac

11
clusters/kube1/default.nix
View File

@@ -22,17 +22,10 @@ let
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
];
};
k8s = import ../../lib/k8s.nix ./.;
cluster = callPackage k8s { inherit settings; };
cluster = callPackage ../../lib/k8s.nix { here = ./.; inherit settings; };
in
{
# k1-0 = cluster.host "10.253.18.109" "k1-0";
# k1-1 = cluster.host "10.253.18.110" "k1-1";
# k1-2 = cluster.host "10.253.18.111" "k1-2";
k1-0 = self:
{
require = [ (cluster.apiserver "10.253.18.109" "k1-0") ];
};
k1-0 = cluster.apiserver "10.253.18.109" "k1-0";
k1-1 = cluster.worker "10.253.18.110" "k1-1";
k1-2 = cluster.worker "10.253.18.111" "k1-2";
}

72
lib/k8s.nix
View File

@@ -1,4 +1,4 @@
here: { pkgs, lib, settings, ...}:
{ pkgs, lib, settings, here, ...}:
with lib;
let
cluster-ca = pkgs.stdenv.mkDerivation {
@@ -19,18 +19,18 @@ let
'';
};
nixos-kubernetes-join-nodes = workers:
let
wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
in
pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
#!/bin/sh
set -e
token=$(cat /var/lib/cfssl/apitoken.secret)
for i in ${wrk}; do
ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
done
'';
#nixos-kubernetes-join-nodes = workers:
# let
# wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
# in
# pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
# #!/bin/sh
# set -e
# token=$(cat /var/lib/cfssl/apitoken.secret)
# for i in ${wrk}; do
# ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
# done
# '';
kube-system-bootstrap = pkgs.stdenv.mkDerivation {
name = "kube-system-bootstrap";
@@ -45,17 +45,19 @@ let
'';
};
install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" ''
install-apitoken = ''
#!${pkgs.bash}/bin/bash
set -e
if [ -d /var/lib/cfssl ]; then
cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
chown cfssl /var/lib/cfssl/apitoken.secret
chmod 600 /var/lib/cfssl/apitoken.secret
fi
chmod 640 /var/lib/cfssl/apitoken.secret
else
mkdir -p /var/lib/kubernetes/secrets
cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
chown root /var/lib/kubernetes/secrets/apitoken.secret
chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
fi
'';
cidr = "10.10.0.0/16";
@@ -94,9 +96,19 @@ rec {
};
environment.systemPackages = [
pkgs.kubernetes-helm
(nixos-kubernetes-join-nodes settings.workers)
# (nixos-kubernetes-join-nodes settings.workers)
kube-system-bootstrap
];
systemd.services.kube-certmgr-apitoken-bootstrap = {
description = "Kubernetes certmgr bootstrapper";
wantedBy = [ "cfssl.service" ];
before = [ "cfssl.target" ];
script = install-apitoken;
serviceConfig = {
RestartSec = "10s";
Restart = "on-failure";
};
};
};
kubeWorker = {
@@ -116,9 +128,22 @@ rec {
};
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
virtualisation.docker.autoPrune.enable = true;
systemd.services.kube-certmgr-apitoken-bootstrap = {
description = "Kubernetes certmgr bootstrapper";
wantedBy = [ "certmgr.service" ];
before = [ "certmgr.target" ];
script = install-apitoken;
serviceConfig = {
RestartSec = "10s";
Restart = "on-failure";
};
};
};
baseNixos = name: {
users.extraUsers.admin.openssh.authorizedKeys.keys =
settings.adminAuthorizedKeys;
imports = [
../nixos/configuration.nix
(here + "/${name}.nix")
@@ -145,19 +170,6 @@ rec {
firewall.allowedTCPPorts = [ 80 443 111 ];
firewall.allowedUDPPorts = [ 111 24007 24008 ];
};
users.extraUsers.admin.openssh.authorizedKeys.keys =
settings.adminAuthorizedKeys;
systemd.services.kube-certmgr-apitoken-bootstrap = {
description = "Kubernetes certmgr bootstrapper";
wantedBy = [ "cfssl.service" ];
before = [ "cfssl.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${install-apitoken}";
};
};
};
apiserver = ip: name: self: