diff --git a/bin/deploy.sh b/bin/deploy.sh index 15005e9..dc246b4 100755 --- a/bin/deploy.sh +++ b/bin/deploy.sh @@ -1,24 +1,27 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." if [ $# = 0 ]; then echo "usage: deploy.sh name ..." exit 1 fi -if [ ! -f $DIR/clusters/$1/default.nix ]; then +if [ ! -f $TOP/clusters/$1/default.nix ]; then echo "error: $1 does not contain a deployment" exit 1 fi -# mkdir -p $1/gcroots +cd $TOP/clusters/$1 -# echo "--- Securing certifiates" -# nix-build -o $1/gcroots/certs $1/build.nix - -echo "--- Updating deployment" -nixops modify -d $1 $DIR/clusters/$1 +nixops list | grep -q $1 +if [ $? = 0 ]; then + echo "--- Updating deployment" + nixops modify -d $1 . +else + echo "--- Creating deployment" + nixops create -d $1 . +fi echo "--- Deploying $1" nixops deploy -d $* --allow-reboot diff --git a/bin/initca.sh b/bin/initca.sh index 1555d00..7ba5bcb 100755 --- a/bin/initca.sh +++ b/bin/initca.sh @@ -1,6 +1,18 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." -# nix-store --add-root `pwd`/gcroots/initca --indirect -r $(nix-instantiate ./initca.nix) -nix-build -o ca $DIR/../lib/initca.nix +if [ "x$1" = "x" ]; then + echo "usage: initca.sh {cluster}" + exit 1 +fi + +ca=$TOP/lib/initca.nix + +cd $TOP/clusters/$1 + +echo "--- Preparing CA certificate" +nix-build -o ca $ca + +echo "--- Safeguarding CA certificate" +nix-store --add-root $(pwd)/ca --indirect -r $(nix-instantiate --add-root $ca) diff --git a/bin/reboot.sh b/bin/reboot.sh new file mode 100755 index 0000000..854165c --- /dev/null +++ b/bin/reboot.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." + +if [ $# = 0 ]; then + echo "usage: reboot.sh cluster " + exit 1 +fi + +nixops reboot -d $1 diff --git a/bin/ssh.sh b/bin/ssh.sh new file mode 100755 index 0000000..f492523 --- /dev/null +++ b/bin/ssh.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." + +if [ $# = 0 ]; then + echo "usage: ssh.sh cluster ..." + exit 1 +fi + +d=$1; shift + +nixops ssh-for-each -d $d $@ diff --git a/bin/teardown.sh b/bin/teardown.sh index e501cf2..b400b1a 100755 --- a/bin/teardown.sh +++ b/bin/teardown.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." +TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." if [ $# != 1 ]; then echo "usage: teardown.sh name" @@ -8,24 +8,30 @@ if [ $# != 1 ]; then fi d=$1 -f=$DIR/clusters/$d/.$d.$$ +f=$TOP/clusters/$d/.$d.$$ -# nixops ssh -d $d ${d}0-0 kubectl delete --all pods -# nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods -# sleep 60 teardown () { - sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $DIR/clusters/$d/default.nix > $f + sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $TOP/clusters/$d/default.nix > $f nixops modify -d $d $f nixops deploy -d $d - # nixops reboot -d $d - # nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl" + nixops reboot -d $d + nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl" rm $f } +cat << EOF -echo "Are you sure you want to tear down $d? (yes/no)" +************************************************************************ +*** *** +*** WARNING: This will irrevokably destroy the running cluster! *** +*** *** +************************************************************************ + +EOF + +echo "Are you sure you want to tear down $d? (YES/no)" read a case $a in - yes) teardown ;; - *) : ;; + YES) teardown ;; + *) echo "Bailing out." ;; esac diff --git a/clusters/kube1/default.nix b/clusters/kube1/default.nix index a7a6e3e..2d1d6e9 100644 --- a/clusters/kube1/default.nix +++ b/clusters/kube1/default.nix @@ -1,10 +1,11 @@ with import {}; let settings = rec { + clusterName = "kube1"; master = "k1-0"; workers = [ "k1-1" "k1-2" ]; masterAddress = "10.253.18.109"; - apiserverAddress = "https://${masterAddress}:8443"; + apiserverAddress = "https://${masterAddress}:4443"; clusterHosts = '' 10.253.18.109 k1-0 kubernetes fs0-2 10.253.18.110 k1-1 diff --git a/clusters/template/default.nix b/clusters/template/default.nix new file mode 100644 index 0000000..87082aa --- /dev/null +++ b/clusters/template/default.nix @@ -0,0 +1,22 @@ +with import {}; +let + settings = rec { + master = "node1"; + workers = [ "node2" ]; + masterAddress = "10.10.10.1"; + apiserverAddress = "https://${masterAddress}:4443"; + clusterHosts = '' + 10.10.10.1 node1 kubernetes + 10.10.10.2 node2 + 10.10.20.1 fs1 + ''; + adminAuthorizedKeys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf admin" + ]; + }; + cluster = callPackage ../../lib/k8s.nix { here = ./.; inherit settings; }; +in +{ + node1 = cluster.apiserver "10.10.10.1" "node1"; + node2 = cluster.worker "10.10.10.2" "node2"; +} diff --git a/clusters/template/node1.nix b/clusters/template/node1.nix new file mode 100644 index 0000000..19a136d --- /dev/null +++ b/clusters/template/node1.nix @@ -0,0 +1,21 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/clusters/template/node2.nix b/clusters/template/node2.nix new file mode 100644 index 0000000..19a136d --- /dev/null +++ b/clusters/template/node2.nix @@ -0,0 +1,21 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/lib/initca.nix b/lib/initca.nix index f12d099..04cb94c 100644 --- a/lib/initca.nix +++ b/lib/initca.nix @@ -1,4 +1,5 @@ -with import {}; +{ pkgs ? import {}, ...}: +with pkgs; let initca' = let diff --git a/lib/k8s.nix b/lib/k8s.nix index ebcd9b5..0979539 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -1,14 +1,7 @@ { pkgs, lib, settings, here, ...}: with lib; let - cluster-ca = pkgs.stdenv.mkDerivation { - name = "cluster-ca"; - src = here + /ca; - buildCommand = '' - mkdir -p $out - cp $src/* $out - ''; - }; + cluster-ca = import ./initca.nix { inherit pgks; }; cfssl-apitoken = pkgs.stdenv.mkDerivation { name = "cfssl-apitoken"; @@ -21,7 +14,7 @@ let kube-system-bootstrap = pkgs.stdenv.mkDerivation { name = "kube-system-bootstrap"; - src = ../kube-system-bootstrap; + src = ./kube-system-bootstrap; buildCommand = '' mkdir -p $out/bin mkdir -p $out/share/kube-system-bootstrap @@ -57,15 +50,19 @@ rec { masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; clusterCidr = cidr; - kubelet.unschedulable = false; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${cluster-ca}/ca"; + kubelet = { + unschedulable = false; + clusterDomain = "${settings.clusterName}.local"; + }; + apiserver = { advertiseAddress = settings.masterAddress; authorizationMode = [ "Node" "RBAC" ]; - securePort = 8443; + securePort = 4443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem"; }; @@ -73,14 +70,14 @@ rec { addons = { dns = { enable = true; - # clusterDomain = "local"; + clusterDomain = "${settings.clusterName}.local"; reconcileMode = "EnsureExists"; }; }; }; networking.firewall = { - allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ]; + allowedTCPPorts = [ 53 5000 8080 4443 ]; #;4053 ]; allowedUDPPorts = [ 53 4053 ]; }; @@ -103,11 +100,11 @@ rec { systemd.services.kube-system-bootstrap = { description = "Kubernetes certmgr bootstrapper"; - after = [ "multi-user.target" ]; + wantedBy = [ "multi-user.target" ]; + after = [ "kubernetes.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = false; - # PATH=$PATH:${pkgs.bash}/bin:${pkgs.kubectl}/bin:${pkgs.kubernetes-helm}/bin:${pkgs.coreutils}/bin Environment = '' PATH=$PATH:/run/current-system/sw/bin ''; @@ -115,7 +112,10 @@ rec { #!${pkgs.bash}/bin/bash set -e if [ ! -f /var/lib/kubernetes/.kube-system-bootstrap.done ]; then - ${pkgs.bash}/bin/bash ${kube-system-bootstrap}/share/kube-system-bootstrap/kube-system-bootstrap ${cluster-ca} + ${pkgs.bash}/bin/bash + d=${kube-system-bootstrap}/share/kube-system-bootstrap + cd $d + $d/kube-system-bootstrap ${cluster-ca} ${settings.clusterName} touch /var/lib/kubernetes/.kube-system-bootstrap.done fi ''; @@ -129,7 +129,9 @@ rec { clusterCidr = cidr; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; + kubelet.clusterDomain = "${settings.clusterName}.local"; }; + networking = { firewall = { enable = true; @@ -143,7 +145,7 @@ rec { systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "certmgr.service" ]; - before = [ "certmgr.target" ]; + # before = [ "certmgr.service" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; @@ -157,7 +159,7 @@ rec { settings.adminAuthorizedKeys; imports = [ - ../nixos/configuration.nix + ./nixos/configuration.nix (here + "/${name}.nix") ]; security.pki.certificateFiles = [ diff --git a/lib/kube-system-bootstrap b/lib/kube-system-bootstrap index c334818..0c5abbc 160000 --- a/lib/kube-system-bootstrap +++ b/lib/kube-system-bootstrap @@ -1 +1 @@ -Subproject commit c334818834160078dfde905212e4c1cc4cf9c314 +Subproject commit 0c5abbce4659f000340cc48efd667843657fdbc8