From b89514eae4a16561da3465948d1bdd9211f238d4 Mon Sep 17 00:00:00 2001
From: Jonas Juselius <jonas.juselius@itpartner.no>
Date: Fri, 30 Oct 2020 15:08:39 +0100
Subject: [PATCH] Modularize k8s configs

---
 lib/default.nix | 132 ++++++++++++++++++++++++++----------------------
 lib/k8s.nix     | 122 ++++++++++++++++++++++++++++++++++----------
 2 files changed, 166 insertions(+), 88 deletions(-)

diff --git a/lib/default.nix b/lib/default.nix
index 1298863..895b93e 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -11,70 +11,76 @@ let
       };
     };
 
-  # hostCerts = builtins.foldl'
-  #   (a: x: a // { ${x.name} = pki.gencert {
-  #       cn = x.name;
-  #       ca = x.ca;
-  #       o = cfg.clusterName;
-  #     };
-  #   }) {} cfg.hosts;
+  hostCerts = builtins.foldl'
+    (a: x: a // { ${x.name} = pki.gencert {
+        cn = x.name;
+        ca = x.ca;
+        o = cfg.clusterName;
+      };
+    }) {} cfg.hosts;
 
-  # mkHost = host: self: {
-  #     deployment.targetHost = host.address;
-  #     require = [
-  #       (baseNixos host.name)
-  #     ];
-  #   };
+  mkHost = host: self: {
+      deployment.targetHost = host.address;
+      require = [
+        (baseNixos host.name)
+      ];
+    };
 
-  # baseDeployment = nodes: attrs:
-  #   let
-  #     hosts =
-  #       builtins.foldl'
-  #         (a: x: a // { ${x.name} = mkHost x _; }) {} nodes;
-  #     hosts' = lib.recursiveUpdate hosts attrs;
-  #     names = builtins.attrNames hosts;
-  #   in
-  #     builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names;
+  baseDeployment = nodes: attrs:
+    let
+      hosts =
+        builtins.foldl'
+          (a: x: a // { ${x.name} = mkHost x _; }) {} nodes;
+      hosts' = lib.recursiveUpdate hosts attrs;
+      names = builtins.attrNames hosts;
+    in
+      builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names;
 
 in {
+  k8s = rec {
+    master = host: self: {
+      deployment.targetHost = host.address;
 
-  # k8s = import ./k8s.nix { inherit pgks lib config; };
+      cluster = cfg // {
+        hostName = host.name;
+        cert = mkCert host.name;
+        k8s.master.enable = true;
+        k8s.node.enable = true;
+      };
 
-  # k8s = rec {
-  #   apiserver = host: self: {
-  #     deployment.targetHost = host.address;
-  #     require = [
-  #       (os.baseNixos host.name)
-  #       k8s.kubeMaster
-  #     ];
-  #   };
+      imports = [ host.hw ./k8s.nix ];
+    };
 
-  #   node = host: self: {
-  #     deployment.targetHost = host.address;
-  #     require = [
-  #       (os.baseNixos host.name)
-  #       k8s.kubeWorker
-  #     ];
-  #   };
+    node = host: self: {
+      deployment.targetHost = host.address;
 
-  #   deployment = masterNode: workerNodes:
-  #     let
-  #       master = { "${master.name}" =  apiserver masterNode; };
-  #     in
-  #       builtins.foldl' (a: x:
-  #         a // { "${x.name}" = mkWorker x; }) master workerNodes;
-  # };
+      cluster = cfg // {
+        hostName = host.name;
+        cert = mkCert host.name;
+        k8s.node.enable = true;
+      };
+
+      imports = [ host.hw ./k8s.nix ];
+    };
+
+    mkDeployment = masterNode: workerNodes:
+      let
+        apiserver = { "${master.name}" =  apiserver masterNode; };
+      in
+        builtins.foldl' (a: x:
+          a // { "${x.name}" = mkWorker x; }) apiserver workerNodes;
+  };
 
   fs = rec {
     mkNode = host: self: {
       deployment.targetHost = host.address;
 
-      imports = [ host.hw ./fs.nix ];
-
       cluster = cfg // {
         hostName = host.name;
         cert = mkCert host.name;
       };
+
+      imports = [ host.hw ./fs.nix ];
     };
 
     mkDeployment = nodes:
@@ -82,19 +88,23 @@ in {
         a // { "${x.name}" = mkNode x; }) {} nodes;
   } ;
 
-  # host = rec {
-  #   node = host: self: {
-  #     deployment.targetHost = host.address;
-  #     require = [
-  #       (os.baseNixos host.name)
-  #     ];
-  #   };
+  host = rec {
+    node = host: self: {
+      deployment.targetHost = host.address;
 
-  #   deployment = masterNode: workerNodes:
-  #     let
-  #       master = { "${master.name}" =  apiserver masterNode; };
-  #     in
-  #       builtins.foldl' (a: x:
-  #         a // { "${x.name}" = mkWorker x; }) master workerNodes;
-  # };
+      cluster = cfg // {
+        hostName = host.name;
+        cert = mkCert host.name;
+      };
+
+      imports = [ host.hw ./os.nix ];
+    };
+
+    mkDeployment = masterNode: workerNodes:
+      let
+        master = { "${master.name}" =  apiserver masterNode; };
+      in
+        builtins.foldl' (a: x:
+          a // { "${x.name}" = mkWorker x; }) master workerNodes;
+  };
 }
diff --git a/lib/k8s.nix b/lib/k8s.nix
index a180afe..3badd52 100644
--- a/lib/k8s.nix
+++ b/lib/k8s.nix
@@ -1,9 +1,13 @@
-{ pkgs, lib, settings, here ? ./., ...}:
-with import ./base.nix { inherit pkgs lib settings here; };
+{ pkgs, lib, config, ...}:
 with lib;
 let
+  cfg = config.cluster;
+
+  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
+
   apiserverAddress = "https://${masterAddress}:4443";
-  masterAddress = settings.master.address;
+
+  masterAddress = cfg.k8s.master.address;
 
   cfssl-apitoken =
     let
@@ -26,13 +30,13 @@ let
     };
 
   kube-system-bootstrap =
-    with settings;
     let
-      worker_nodes = pkgs.writeText "kube-worker-nodes" (
+      workerNodes = pkgs.writeText "kube-worker-nodes" (
         builtins.foldl' (a: x:
           a + "  - ${x.address}\n"
-        ) "" settings.workers);
-      grafana_ldap = pkgs.writeText "grafana-ldap.toml" grafana_ldap_toml;
+        ) "" cfg.k8s.nodes);
+      grafanaLdap = pkgs.writeText "grafana-ldap.toml"
+          cfg.k8s.bootstrap.grafana_ldap_toml;
     in
     pkgs.stdenv.mkDerivation {
       name = "kube-system-bootstrap";
@@ -45,16 +49,16 @@ let
         mkdir -p $share/charts
 
         export bash="${pkgs.bash}"
-        export apiserver="${settings.master.name}"
-        export apiserverAddress="${settings.master.address}"
+        export apiserver="${cfg.k8s.master.name}"
+        export apiserverAddress="${cfg.k8s.master.address}"
         export initca="${pki.initca}"
-        export cluster="${clusterName}"
-        export fileserver="${fileserver}"
-        export acme_email="${acme_email}"
-        export grafana_smtp_user="$(echo -n ${grafana_smtp_user} | base64 -w0)"
-        export grafana_smtp_password="$(echo -n ${grafana_smtp_password} | base64 -w0)"
-        export grafana_ldap_toml="$(cat ${grafana_ldap} | base64 -w0)"
-        export workers="$(cat ${worker_nodes})"
+        export cluster="${cfg.clusterName}"
+        export fileserver="${cfg.k8s.fileserver}"
+        export acme_email="${cfg.k8s.bootrstrap.acme_email}"
+        export grafana_smtp_user="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_user} | base64 -w0)"
+        export grafana_smtp_password="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_password} | base64 -w0)"
+        export grafana_ldap_toml="$(cat ${grafanaLdap} | base64 -w0)"
+        export workers="$(cat ${workerNodes})"
 
         substituteAll $src/bin/initial-kube-system-bootstrap $share/bin/initial-kube-system-bootstrap
         chmod 755 $share/bin/initial-kube-system-bootstrap
@@ -97,14 +101,14 @@ let
     services.kubernetes = {
       roles = [ "master" ];
       inherit apiserverAddress;
-      masterAddress = settings.master.name;
-      clusterCidr = settings.cidr;
+      masterAddress = cfg.k8s.master.name;
+      clusterCidr = cfg.k8s.cidr;
       pki.genCfsslCACert = false;
       pki.genCfsslAPIToken = false;
       pki.caCertPathPrefix = "${pki.initca}/ca";
 
       kubelet = {
-        clusterDomain = "${settings.clusterName}.local";
+        clusterDomain = "${cfg.clusterName}.local";
       };
 
       apiserver = {
@@ -114,7 +118,7 @@ let
         securePort = 4443;
         insecurePort = 8080;
         extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
-        extraSANs = settings.master.extraSANs;
+        extraSANs = cfg.k8s.master.extraSANs;
         # verbosity = 4;
       };
 
@@ -129,7 +133,7 @@ let
       addons = {
         dns = {
           enable = true;
-          clusterDomain = "${settings.clusterName}.local";
+          clusterDomain = "${cfg.clusterName}.local";
           reconcileMode = "EnsureExists";
         };
       };
@@ -168,13 +172,13 @@ let
     };
   };
 
-  kubeWorker = {
+  kubeNode = {
     services.kubernetes = rec {
       roles = [ "node" ];
       inherit apiserverAddress;
-      masterAddress = settings.master.name;
-      clusterCidr = settings.cidr;
-      kubelet.clusterDomain = "${settings.clusterName}.local";
+      masterAddress = cfg.k8s.master.name;
+      clusterCidr = cfg.k8s.cidr;
+      kubelet.clusterDomain = "${cfg.clusterName}.local";
     };
 
     networking = {
@@ -199,5 +203,69 @@ let
     };
   };
 
-in
-  deployment
+in {
+  options.cluster.k8s = {
+    nodes = mkOption {
+      type = types.attrs;
+      default = {};
+    };
+
+    fileserver = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+    };
+
+    cidr = mkOption {
+      type = types.str;
+      default = "10.11.0.0/16";
+    };
+
+    master = {
+      enable = mkEnableOption "Enable kubernetes master node";
+
+      address = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+      };
+
+      name = mkOption {
+        type = types.str;
+        default = "apiserver";
+      };
+    };
+
+    node = {
+      enable = mkEnableOption "Enable kubernetes";
+    };
+
+    bootstrap = {
+      acme_email = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_smtp_user = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_smtp_password = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_ldap = mkOption {
+        type = types.str;
+        default = "";
+      };
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.k8s.master.enable kubeMaster)
+    (mkIf cfg.k8s.node.enable kubeNode)
+  ];
+
+  imports = [ ./os.nix ];
+}
+