Multiple improvements and refactorizations

lib/k8s.nix

@@ -1,7 +1,11 @@
-{ pkgs, lib, settings, here, ...}:
+{ pkgs, lib, settings, here ? "", ...}:
 with lib;
 let
-  cluster-ca = import ./initca.nix { inherit pgks; };
+  apiserverAddress = "https://${masterAddress}:4443";
+  masterAddress = settings.master.address;
+  initca = settings.initca;
+
+  cluster-ca = import ./initca.nix { inherit pgks initca; };
 
   cfssl-apitoken = pkgs.stdenv.mkDerivation {
     name = "cfssl-apitoken";
@@ -12,30 +16,50 @@ let
     '';
   };
 
-  kube-system-bootstrap = pkgs.stdenv.mkDerivation {
-    name = "kube-system-bootstrap";
-    src = ./kube-system-bootstrap;
-    buildCommand = ''
-     mkdir -p $out
-     cp -r $src/* $out
-     cp ${here}/bootstrap.conf $out/${settings.clusterName}.conf
-    '';
-  };
+  kube-system-bootstrap =
+    with settings;
+    let
+      worker_nodes = pkgs.writeText "worker-nodes.txt" (
+        builtins.foldl' (a: x:
+          a + "  - ${x.address}\n"
+        ) "" settings.workers);
+      grafana_ldap = pkgs.writeText "grafana-ldap.toml" grafana_ldap_toml;
+    in
+    pkgs.stdenv.mkDerivation {
+      name = "bootstrap-kube-system";
+      src = ../bootstrap;
+      buildCommand = ''
+        mkdir -p $out/bin
+        mkdir -p $out/share/kube-system-bootstrap/config
+        mkdir -p $out/share/kube-system-bootstrap/charts
 
-  bootstrap-kube-system-sh = pkgs.writeScriptBin "bootstrap-kube-system.sh" ''
-    #!${pkgs.bash}/bin/bash
-    cd ${kube-system-bootstrap}
-    ${pkgs.bash}/bin/bash ./kube-system-bootstrap ${cluster-ca} ${settings.clusterName}
-  '';
+        export bash="${pkgs.bash}"
+        export apiserver="${settings.master.address}"
+        export initca="${initca}"
+        export cluster="${clusterName}"
+        export fileserver="${fileserver}"
+        export acme_email="${acme_email}"
+        export grafana_smtp_user="$(echo -n ${grafana_smtp_user} | base64 -w0)"
+        export grafana_smtp_password="$(echo -n ${grafana_smtp_password} | base64 -w0)"
+        export grafana_ldap_toml="$(cat ${grafana_ldap} | base64 -w0)"
+        export workers="$(cat ${worker_nodes})"
 
-  kube-scripts = pkgs.stdenv.mkDerivation {
-    name = "kube-scripts";
-    buildCommand = ''
-     mkdir -p $out/bin
-     cd $out/bin
-     ln -s ${kube-system-bootstrap}/bin/* .
-    '';
-  };
+        substituteAll $src/kube-system-bootstrap $out/bin/bootstrap-kube-system
+        chmod 755 $out/bin/bootstrap-kube-system
+
+        cd $src/config
+        for i in *; do
+          substituteAll $i $out/share/kube-system-bootstrap/config/$i
+        done
+
+        cd $src/charts
+        for i in *; do
+          substituteAll $i $out/share/kube-system-bootstrap/charts/$i
+        done
+
+        cp $src/bin/* $out/bin
+      '';
+    };
 
   install-apitoken = ''
       #!${pkgs.bash}/bin/bash
@@ -52,34 +76,38 @@ let
       fi
     '';
 
-  cidr = "10.10.0.0/16";
-in
-rec {
   kubeMaster = {
     services.cfssl.ca = "${cluster-ca}/ca.pem";
     services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
     services.kubernetes = {
       roles = [ "master" ];
-      masterAddress = settings.master;
-      apiserverAddress = settings.apiserverAddress;
-      clusterCidr = cidr;
+      inherit apiserverAddress masterAddress;
+      clusterCidr = settings.cidr;
       pki.genCfsslCACert = false;
       pki.genCfsslAPIToken = false;
       pki.caCertPathPrefix = "${cluster-ca}/ca";
 
       kubelet = {
-        unschedulable = false;
         clusterDomain = "${settings.clusterName}.local";
       };
 
       apiserver = {
-        advertiseAddress = settings.masterAddress;
+        advertiseAddress = masterAddress;
         authorizationMode = [ "Node" "RBAC" ];
+        allowPrivileged = true;
         securePort = 4443;
         insecurePort = 8080;
         extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
       };
 
+      controllerManager = {
+        bindAddress = masterAddress;
+        extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
+      };
+
+      scheduler.address = masterAddress;
+
+      addonManager.enable = true;
       addons = {
         dns = {
           enable = true;
@@ -89,16 +117,19 @@ rec {
       };
     };
 
+    services.etcd = {
+      listenClientUrls = [ "https://${masterAddress}:2379" ];
+    };
+
     networking.firewall = {
-      allowedTCPPorts = [ 53 5000 8080 4443 ]; #;4053 ];
+      allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
       allowedUDPPorts = [ 53 4053 ];
     };
 
     environment.systemPackages = [
       pkgs.kubernetes-helm
       pkgs.kubectl
-      kube-scripts
-      bootstrap-kube-system-sh
+      kube-system-bootstrap
     ];
 
     systemd.services.kube-certmgr-apitoken-bootstrap = {
@@ -116,9 +147,8 @@ rec {
   kubeWorker = {
     services.kubernetes = rec {
       roles = [ "node" ];
-      clusterCidr = cidr;
-      masterAddress = settings.master;
-      apiserverAddress = settings.apiserverAddress;
+      inherit apiserverAddress masterAddress;
+      clusterCidr = settings.cidr;
       kubelet.clusterDomain = "${settings.clusterName}.local";
     };
 
@@ -148,6 +178,11 @@ rec {
     users.extraUsers.admin.openssh.authorizedKeys.keys =
      settings.adminAuthorizedKeys;
 
+    boot.kernel.sysctl = {
+      "kernel.mm.transparent_hugepage.enabled" = "never";
+      "net.core.somaxconn" = "512";
+    };
+
     imports = [
       ./nixos/configuration.nix
       (here + "/${name}.nix")
@@ -166,10 +201,6 @@ rec {
     networking = {
       hostName = name;
       extraHosts = settings.clusterHosts;
-      # nameservers = [ masterAddress ];
-      # dhcpcd.extraConfig = ''
-      #  static domain_name_servers=${masterAddress}
-      # '';
       firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
       firewall.allowedTCPPorts = [ 80 443 111 ];
       firewall.allowedUDPPorts = [ 111 24007 24008 ];
@@ -179,29 +210,33 @@ rec {
     ];
   };
 
-  apiserver = ip: name: self:
+  mkApiServer = host: self:
     {
-      deployment.targetHost = ip;
+      deployment.targetHost = host.address;
       require = [
-        (baseNixos name)
+        (baseNixos host.name)
         kubeMaster
       ];
     };
 
-  worker = ip: name: self:
+  mkWorker = host: self:
     {
-      deployment.targetHost = ip;
+      deployment.targetHost = host.address;
       require = [
-        (baseNixos name)
+        (baseNixos host.name)
         kubeWorker
       ];
     };
 
-  host = ip: name: self:
+  mkHost = host: self:
     {
-      deployment.targetHost = ip;
+      deployment.targetHost = host.address;
       require = [
-        (baseNixos name)
+        (baseNixos host.name)
       ];
     };
-}
+
+  master = { "${settings.master.name}" = mkApiServer settings.master; };
+in
+  builtins.foldl'
+    (a: x: a // { "${x.name}" = mkWorker x; }) master settings.workers