Reorganize project

modules/k8s.nix

@@ -0,0 +1,283 @@
+{ pkgs, lib, config, ...}:
+with lib;
+let
+  cfg = config.cluster;
+
+  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
+
+  masterAddress = cfg.k8s.master.address;
+
+  apiserverAddress = "https://${masterAddress}:4443";
+
+  cfssl-apitoken =
+    let
+      apitoken = pkgs.stdenv.mkDerivation {
+        name = "apitoken";
+        buildCommand = ''
+          head -c ${toString (32 / 2)} /dev/urandom | \
+          od -An -t x | tr -d ' ' > $out
+          chmod 400 $out
+        '';
+      };
+    in
+    # make ca derivation sha depend on initca cfssl output
+    pkgs.stdenv.mkDerivation {
+      name = "cfssl-apitoken";
+      src = apitoken;
+      buildCommand = ''
+        cp $src $out
+      '';
+    };
+
+  kube-system-bootstrap =
+    let
+      workerNodes = pkgs.writeText "kube-worker-nodes" (
+        builtins.foldl' (a: x:
+          a + "  - ${x.address}\n") "" cfg.k8s.nodes);
+      grafanaLdap = pkgs.writeText "grafana-ldap.toml"
+          cfg.k8s.bootstrap.grafana_ldap_toml;
+    in
+    pkgs.stdenv.mkDerivation {
+      name = "kube-system-bootstrap";
+      src = ../bootstrap;
+      buildCommand = ''
+        share=$out/share/kube-system-bootstrap
+        mkdir -p $out/bin
+        mkdir -p $share/bin
+        mkdir -p $share/config
+        mkdir -p $share/charts
+
+        export bash="${pkgs.bash}"
+        export apiserver="${cfg.k8s.master.name}"
+        export apiserverAddress="${cfg.k8s.master.address}"
+        export initca="${pki.initca}"
+        export cluster="${cfg.clusterName}"
+        export fileserver="${cfg.k8s.fileserver}"
+        export acme_email="${cfg.k8s.bootstrap.acme_email}"
+        export grafana_smtp_user="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_user} | base64 -w0)"
+        export grafana_smtp_password="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_password} | base64 -w0)"
+        export grafana_ldap_toml="$(cat ${grafanaLdap} | base64 -w0)"
+        export workers="$(cat ${workerNodes})"
+
+        substituteAll $src/bin/initial-kube-system-bootstrap $share/bin/initial-kube-system-bootstrap
+        chmod 755 $share/bin/initial-kube-system-bootstrap
+
+        substituteAll $src/copy-kube-system-bootstrap $out/bin/copy-kube-system-bootstrap
+        chmod 755 $out/bin/copy-kube-system-bootstrap
+
+        cd $src/config
+        for i in *; do
+          substituteAll $i $share/config/$i
+        done
+
+        cd $src/charts
+        for i in *; do
+          substituteAll $i $share/charts/$i
+        done
+
+        cp $src/bin/* $share/bin
+      '';
+    };
+
+  install-apitoken = ''
+      #!${pkgs.bash}/bin/bash
+      set -e
+      if [ -d /var/lib/cfssl ]; then
+        cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
+        chown cfssl /var/lib/cfssl/apitoken.secret
+        chmod 640 /var/lib/cfssl/apitoken.secret
+      else
+        mkdir -p /var/lib/kubernetes/secrets
+        cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
+        chown root /var/lib/kubernetes/secrets/apitoken.secret
+        chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
+      fi
+    '';
+
+  kubeMaster = {
+    services.cfssl.ca = pki.ca.cert;
+    services.cfssl.caKey = pki.ca.key;
+    services.kubernetes = {
+      roles = [ "master" ];
+      inherit apiserverAddress;
+      masterAddress = cfg.k8s.master.name;
+      clusterCidr = cfg.k8s.cidr;
+      pki.genCfsslCACert = false;
+      pki.genCfsslAPIToken = false;
+      pki.caCertPathPrefix = "${pki.initca}/ca";
+
+      kubelet = {
+        clusterDomain = "${cfg.clusterName}.local";
+      };
+
+      apiserver = {
+        advertiseAddress = masterAddress;
+        authorizationMode = [ "Node" "RBAC" ];
+        allowPrivileged = true;
+        securePort = 4443;
+        insecurePort = 8080;
+        extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
+        extraSANs = cfg.k8s.master.extraSANs;
+        # verbosity = 4;
+      };
+
+      controllerManager = {
+        bindAddress = masterAddress;
+        extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
+      };
+
+      scheduler.address = masterAddress;
+
+      addonManager.enable = true;
+      addons = {
+        dns = {
+          enable = true;
+          clusterDomain = "${cfg.clusterName}.local";
+          reconcileMode = "EnsureExists";
+        };
+      };
+    };
+
+    services.etcd = {
+      listenClientUrls = [ "https://${masterAddress}:2379" ];
+    };
+
+    networking.firewall = {
+      allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
+      allowedUDPPorts = [ 53 4053 ];
+    };
+
+    environment.systemPackages = [
+      pkgs.kubernetes-helm
+      pkgs.kubectl
+      kube-system-bootstrap
+    ];
+
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "cfssl.service" ];
+      before = [ "cfssl.target" ];
+      script = install-apitoken;
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
+    systemd.services.cfssl-restart  = {
+      enable = true;
+      startAt  = "00/6:00";
+      description = "Restrart cfssl which regularly locks up";
+      script = "systemctl restart cfssl.service";
+    };
+  };
+
+  kubeNode = {
+    services.kubernetes = rec {
+      roles = [ "node" ];
+      inherit apiserverAddress;
+      masterAddress = cfg.k8s.master.name;
+      clusterCidr = cfg.k8s.cidr;
+      kubelet.clusterDomain = "${cfg.clusterName}.local";
+    };
+
+    networking = {
+      firewall = {
+        enable = true;
+        allowedTCPPorts = [ 4194 10250 ];
+        allowedUDPPorts = [ 53 ];
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
+      };
+    };
+    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
+    virtualisation.docker.autoPrune.enable = true;
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "certmgr.service" ];
+      before = [ "certmgr.service" ];
+      script = install-apitoken;
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
+  };
+
+in {
+  options.cluster.k8s = {
+    enable = mkEnableOption "Enable kubernetes";
+    nodes = mkOption {
+      type = types.listOf types.attrs;
+      default = [];
+    };
+
+    fileserver = mkOption {
+      type = types.str;
+      default = null;
+    };
+
+    cidr = mkOption {
+      type = types.str;
+      default = "10.0.0.0/16";
+    };
+
+    master = {
+      enable = mkEnableOption "Enable kubernetes master node";
+
+      address = mkOption {
+        type = types.str;
+        default = null;
+      };
+
+      name = mkOption {
+        type = types.str;
+        default = null;
+      };
+
+      extraSANs = mkOption {
+        type = types.listOf types.str;
+        default = [];
+      };
+
+      hw = mkOption {
+        type = types.path;
+        default = null;
+      };
+    };
+
+    node = {
+      enable = mkEnableOption "Enable kubernetes";
+    };
+
+    bootstrap = {
+      acme_email = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_smtp_user = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_smtp_password = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_ldap_toml = mkOption {
+        type = types.str;
+        default = "";
+      };
+    };
+  };
+
+  config = mkIf cfg.k8s.enable (
+    mkMerge [
+      (mkIf cfg.k8s.master.enable kubeMaster)
+      (mkIf cfg.k8s.node.enable kubeNode)
+    ]
+  );
+
+  imports = [ ./os.nix ];
+}
+