Make things more configurable

bin/deploy.sh

@@ -1,24 +1,27 @@
 #!/usr/bin/env bash
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
+TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
 
 if [ $# = 0 ]; then
     echo "usage: deploy.sh name ..."
     exit 1
 fi
 
-if [ ! -f $DIR/clusters/$1/default.nix ]; then
+if [ ! -f $TOP/clusters/$1/default.nix ]; then
     echo "error: $1 does not contain a deployment"
     exit 1
 fi
 
-# mkdir -p $1/gcroots
-
-# echo "--- Securing certifiates"
-# nix-build -o $1/gcroots/certs $1/build.nix
+cd $TOP/clusters/$1
 
+nixops list | grep -q $1
+if [ $? = 0 ]; then
     echo "--- Updating deployment"
-nixops modify -d $1 $DIR/clusters/$1
+    nixops modify -d $1 .
+else
+    echo "--- Creating deployment"
+    nixops create -d $1 .
+fi
 
 echo "--- Deploying $1"
 nixops deploy -d $* --allow-reboot

bin/initca.sh

@@ -1,6 +1,18 @@
 #!/usr/bin/env bash
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
 
-# nix-store --add-root `pwd`/gcroots/initca --indirect -r $(nix-instantiate ./initca.nix)
-nix-build -o ca $DIR/../lib/initca.nix
+if [ "x$1" = "x" ]; then
+    echo "usage: initca.sh {cluster}"
+    exit 1
+fi
+
+ca=$TOP/lib/initca.nix
+
+cd $TOP/clusters/$1
+
+echo "--- Preparing CA certificate"
+nix-build -o ca $ca
+
+echo "--- Safeguarding CA certificate"
+nix-store --add-root $(pwd)/ca --indirect -r $(nix-instantiate --add-root $ca)

bin/reboot.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
+
+if [ $# = 0 ]; then
+    echo "usage: reboot.sh cluster "
+    exit 1
+fi
+
+nixops reboot -d $1

bin/ssh.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
+
+if [ $# = 0 ]; then
+    echo "usage: ssh.sh cluster ..."
+    exit 1
+fi
+
+d=$1; shift
+
+nixops ssh-for-each -d $d $@

bin/teardown.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
+TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
 
 if [ $# != 1 ]; then
     echo "usage: teardown.sh name"
@@ -8,24 +8,30 @@ if [ $# != 1 ]; then
 fi
 
 d=$1
-f=$DIR/clusters/$d/.$d.$$
+f=$TOP/clusters/$d/.$d.$$
 
-# nixops ssh -d $d ${d}0-0 kubectl delete --all pods
-# nixops ssh -d $d ${d}0-0 kubectl --namespace kube-system delete --all pods
-# sleep 60
 teardown () {
-    sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $DIR/clusters/$d/default.nix > $f
+    sed -s 's/cluster.\(apiserver\|worker\)/cluster.host/' $TOP/clusters/$d/default.nix > $f
     nixops modify -d $d $f
     nixops deploy -d $d
-    # nixops reboot -d $d
-    # nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
+    nixops reboot -d $d
+    nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
     rm $f
 }
+cat << EOF
 
-echo "Are you sure you want to tear down $d? (yes/no)"
+************************************************************************
+***                                                                  ***
+*** WARNING: This will irrevokably destroy the running cluster!      ***
+***                                                                  ***
+************************************************************************
+
+EOF
+
+echo "Are you sure you want to tear down $d? (YES/no)"
 read a
 case $a in
-    yes) teardown ;;
-    *) : ;;
+    YES) teardown ;;
+    *) echo "Bailing out." ;;
 esac
 

clusters/kube1/default.nix

@@ -1,10 +1,11 @@
 with import <nixpkgs> {};
 let
   settings = rec {
+    clusterName = "kube1";
     master = "k1-0";
     workers = [ "k1-1" "k1-2" ];
     masterAddress = "10.253.18.109";
-    apiserverAddress = "https://${masterAddress}:8443";
+    apiserverAddress = "https://${masterAddress}:4443";
     clusterHosts = ''
       10.253.18.109 k1-0 kubernetes fs0-2
       10.253.18.110 k1-1

clusters/template/default.nix

@@ -0,0 +1,22 @@
+with import <nixpkgs> {};
+let
+  settings = rec {
+    master = "node1";
+    workers = [ "node2" ];
+    masterAddress = "10.10.10.1";
+    apiserverAddress = "https://${masterAddress}:4443";
+    clusterHosts = ''
+      10.10.10.1 node1 kubernetes
+      10.10.10.2 node2
+      10.10.20.1 fs1
+    '';
+    adminAuthorizedKeys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf admin"
+    ];
+  };
+  cluster = callPackage ../../lib/k8s.nix { here = ./.; inherit settings; };
+in
+{
+  node1 = cluster.apiserver "10.10.10.1" "node1";
+  node2 = cluster.worker "10.10.10.2" "node2";
+}

clusters/template/node1.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

clusters/template/node2.nix

@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}

lib/initca.nix

@@ -1,4 +1,5 @@
-with import <nixpkgs> {};
+{ pkgs ? import <nixpkgs> {}, ...}:
+with pkgs;
 let
   initca' =
     let

lib/k8s.nix

@@ -1,14 +1,7 @@
 { pkgs, lib, settings, here, ...}:
 with lib;
 let
-  cluster-ca = pkgs.stdenv.mkDerivation {
-    name = "cluster-ca";
-    src = here + /ca;
-    buildCommand = ''
-     mkdir -p $out
-     cp $src/* $out
-    '';
-  };
+  cluster-ca = import ./initca.nix { inherit pgks; };
 
   cfssl-apitoken = pkgs.stdenv.mkDerivation {
     name = "cfssl-apitoken";
@@ -21,7 +14,7 @@ let
 
   kube-system-bootstrap = pkgs.stdenv.mkDerivation {
     name = "kube-system-bootstrap";
-    src = ../kube-system-bootstrap;
+    src = ./kube-system-bootstrap;
     buildCommand = ''
      mkdir -p $out/bin
      mkdir -p $out/share/kube-system-bootstrap
@@ -57,15 +50,19 @@ rec {
       masterAddress = settings.master;
       apiserverAddress = settings.apiserverAddress;
       clusterCidr = cidr;
-      kubelet.unschedulable = false;
       pki.genCfsslCACert = false;
       pki.genCfsslAPIToken = false;
       pki.caCertPathPrefix = "${cluster-ca}/ca";
 
+      kubelet = {
+        unschedulable = false;
+        clusterDomain = "${settings.clusterName}.local";
+      };
+
       apiserver = {
         advertiseAddress = settings.masterAddress;
         authorizationMode = [ "Node" "RBAC" ];
-        securePort = 8443;
+        securePort = 4443;
         insecurePort = 8080;
         extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
       };
@@ -73,14 +70,14 @@ rec {
       addons = {
         dns = {
           enable = true;
-          # clusterDomain = "local";
+          clusterDomain = "${settings.clusterName}.local";
           reconcileMode = "EnsureExists";
         };
       };
     };
 
     networking.firewall = {
-      allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
+      allowedTCPPorts = [ 53 5000 8080 4443 ]; #;4053 ];
       allowedUDPPorts = [ 53 4053 ];
     };
 
@@ -103,11 +100,11 @@ rec {
 
     systemd.services.kube-system-bootstrap = {
       description = "Kubernetes certmgr bootstrapper";
-      after = [ "multi-user.target" ];
+      wantedBy = [ "multi-user.target" ];
+      after = [ "kubernetes.target" ];
       serviceConfig = {
         Type = "oneshot";
         RemainAfterExit = false;
-        # PATH=$PATH:${pkgs.bash}/bin:${pkgs.kubectl}/bin:${pkgs.kubernetes-helm}/bin:${pkgs.coreutils}/bin
         Environment = ''
           PATH=$PATH:/run/current-system/sw/bin
         '';
@@ -115,7 +112,10 @@ rec {
           #!${pkgs.bash}/bin/bash
           set -e
           if [ ! -f /var/lib/kubernetes/.kube-system-bootstrap.done ]; then
-            ${pkgs.bash}/bin/bash ${kube-system-bootstrap}/share/kube-system-bootstrap/kube-system-bootstrap ${cluster-ca}
+            ${pkgs.bash}/bin/bash
+            d=${kube-system-bootstrap}/share/kube-system-bootstrap
+            cd $d
+            $d/kube-system-bootstrap ${cluster-ca} ${settings.clusterName}
             touch /var/lib/kubernetes/.kube-system-bootstrap.done
           fi
         '';
@@ -129,7 +129,9 @@ rec {
       clusterCidr = cidr;
       masterAddress = settings.master;
       apiserverAddress = settings.apiserverAddress;
+      kubelet.clusterDomain = "${settings.clusterName}.local";
     };
+
     networking = {
       firewall = {
         enable = true;
@@ -143,7 +145,7 @@ rec {
     systemd.services.kube-certmgr-apitoken-bootstrap = {
       description = "Kubernetes certmgr bootstrapper";
       wantedBy = [ "certmgr.service" ];
-      before = [ "certmgr.target" ];
+      # before = [ "certmgr.service" ];
       script = install-apitoken;
       serviceConfig = {
         RestartSec = "10s";
@@ -157,7 +159,7 @@ rec {
      settings.adminAuthorizedKeys;
 
     imports = [
-      ../nixos/configuration.nix
+      ./nixos/configuration.nix
       (here + "/${name}.nix")
     ];
     security.pki.certificateFiles = [