Update k0 to new-style nix modules

modules.bak/default.nix

@@ -0,0 +1,130 @@
+{ pkgs, cluster, customize ? {}, extraConfig ? {}, lib, config, ... }:
+with lib;
+let
+  cfg = cluster;
+
+  mkSANs = host: [
+    host.name
+    host.address
+    "127.0.0.1"
+  ];
+
+  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
+
+  mkCert = host: {
+    ${host.name} = pki.gencert {
+      cn = host.name;
+      ca = cfg.ca;
+      o = cfg.clusterName;
+      hosts = [
+        host.name
+        "${host.name}.${cfg.domain}"
+        host.address
+        "127.0.0.1"
+      ];
+    };
+  };
+in {
+  k8s = rec {
+    apiserver = host: self: {
+      deployment.targetHost = host.address;
+
+      inherit customize;
+
+      cluster = mkMerge [
+        cfg
+        {
+          hostName = host.name;
+          address = host.address;
+          k8s.master.enable = true;
+          k8s.node.enable = true;
+        }
+      ];
+
+      services.kubernetes.kubelet.extraSANs = mkSANs host;
+
+      imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
+    };
+
+    node = host: self: {
+      deployment.targetHost = host.address;
+
+      inherit customize;
+
+      cluster = mkMerge [
+        cfg
+        {
+          hostName = host.name;
+          address = host.address;
+          k8s.node.enable = true;
+        }
+      ];
+
+      services.kubernetes.kubelet.extraSANs = mkSANs host;
+
+      imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
+    };
+
+    mkDeployment = top:
+      let
+        master = cfg.k8s.master // { hw = "${top}/${master.name}.nix"; };
+        nodes = cfg.k8s.nodes;
+        server = { "${master.name}" = apiserver master; };
+      in
+        builtins.foldl' (a: x:
+          a // {
+            "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
+          }) server nodes;
+  };
+
+  fs = rec {
+    node = host: self: {
+      deployment.targetHost = host.address;
+
+      inherit customize;
+
+      cluster = mkMerge [
+        { fs.nfs.enable = mkDefault true; }
+        cfg
+        {
+          hostName = host.name;
+          address = host.address;
+          cert = mkCert host.name;
+        }
+      ];
+
+      imports = [ host.hw extraConfig ./modules.nix ];
+    };
+
+    mkDeployment = top: nodes:
+      builtins.foldl' (a: x:
+        a // {
+          "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
+        }) {} nodes;
+  } ;
+
+  host = rec {
+    node = host: self: {
+      deployment.targetHost = host.address;
+
+      inherit customize;
+
+      cluster = mkMerge [
+        cfg
+        {
+          hostName = host.name;
+          address = host.address;
+          cert = mkCert host.name;
+        }
+      ];
+
+      imports = [ host.hw extraConfig ./modules.nix ];
+    };
+
+    mkDeployment = top: nodes:
+      builtins.foldl' (a: x:
+        a // {
+          "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
+        }) {} nodes;
+    };
+}

modules.bak/fs.nix

@@ -0,0 +1,67 @@
+{ pkgs, lib, config, ... }:
+with lib;
+let
+  cfg = config.cluster.fs;
+
+  cert = cfg.cert;
+
+  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
+
+  common = {
+    boot.kernelModules = [
+      "dm_snapshot"
+      "dm_mirror"
+      "dm_thin_pool"
+    ];
+
+    networking = {
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 111 2049 ];
+      firewall.allowedUDPPorts = [ 111 2049 24007 24008 ];
+    };
+
+    environment.systemPackages = [ pkgs.lvm2 ];
+  };
+
+  glusterfs = {
+    services.glusterfs = {
+      enable = true;
+      tlsSettings = {
+        caCert = pki.ca.cert;
+        tlsKeyPath = cert.key;
+        tlsPem = cert.cert;
+      };
+    };
+  };
+
+  nfs = {
+    services.nfs.server = {
+      enable = true;
+      exports = cfg.nfs.exports;
+    };
+  };
+in {
+  options.cluster.fs = {
+    enable = mkEnableOption "Enable nfs fileserver";
+
+    nfs = {
+      enable = mkEnableOption "Enable nfs fileserver";
+      exports = mkOption {
+        type = types.str;
+        default = "";
+      };
+    };
+
+    glusterfs.enable = mkEnableOption "Enable glusterfs fileserver";
+  };
+
+  config = mkIf cfg.enable (
+    mkMerge [
+      common
+      (mkIf cfg.nfs.enable nfs)
+      (mkIf cfg.glusterfs.enable glusterfs)
+    ]
+  );
+
+  imports = [ ./os.nix ];
+}

modules.bak/initca.nix

@@ -0,0 +1,42 @@
+{
+  pkgs ? import <nixpkgs> {},
+  ca ? null,
+  name ? "ca",
+  algo ? "rsa",
+  hosts ? [],
+  ...}:
+with pkgs;
+let
+  ca_csr = pkgs.writeText "${name}-csr.json" (builtins.toJSON {
+      inherit hosts;
+      CN = "${name}";
+      key = {
+        inherit algo;
+        size = if algo == "ecdsa" then 256 else 2048;
+      };
+      names = [
+        {
+          CN = "${name}";
+          O = "NixOS";
+          OU = "${name}.pki.caSpec";
+          L = "generated";
+        }
+      ];
+    }
+  );
+  ca' =
+      pkgs.runCommand "initca" {
+        buildInputs = [ pkgs.cfssl ];
+      } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca;
+           mkdir -p $out; cp *.pem $out '';
+   initca = if ca != null then ca else ca';
+in
+  # make ca derivation sha depend on initca cfssl output
+  pkgs.stdenv.mkDerivation {
+    inherit name;
+    src = initca;
+    buildCommand = ''
+      mkdir -p $out;
+      cp -r $src/* $out
+    '';
+  }

modules.bak/k8s.nix

@@ -0,0 +1,384 @@
+{ pkgs, lib, config, ...}:
+with lib;
+let
+  cfg = config.cluster;
+
+  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
+
+  masterAddress = cfg.k8s.master.address;
+
+  apiserverAddress = "https://${masterAddress}:4443";
+
+  cfssl-apitoken =
+    let
+      apitoken = pkgs.stdenv.mkDerivation {
+        name = "apitoken";
+        buildCommand = ''
+          head -c ${toString (32 / 2)} /dev/urandom | \
+          od -An -t x | tr -d ' ' > $out
+          chmod 400 $out
+        '';
+      };
+    in
+    # make ca derivation sha depend on initca cfssl output
+    pkgs.stdenv.mkDerivation {
+      name = "cfssl-apitoken";
+      src = apitoken;
+      buildCommand = ''
+        cp $src $out
+      '';
+    };
+
+    cluster-scripts =
+      let
+        first = builtins.head cfg.k8s.ingressNodes;
+        rest =  builtins.tail cfg.k8s.ingressNodes;
+        ingressNodes = builtins.foldl' (a: x:
+          a + ",${x}") first rest;
+        ingressReplicaCount =
+           builtins.toString (builtins.length cfg.k8s.ingressNodes);
+        show-kubernetes-charts-config = ''
+            #!/usr/bin/env bash
+            cat << EOF
+            # Generated by show-kubernetes-charts-config
+            #   $(date)
+            # Charts in ${kubernetes-charts}/share/kubernetes-charts
+
+            vars=(
+                initca="${pki.initca}"
+                apiserver="${cfg.k8s.master.name}"
+                cluster="${cfg.clusterName}"
+                ingress_nodes="${ingressNodes}"
+                ingress_replica_count="${ingressReplicaCount}"
+                fileserver="${cfg.k8s.fileserver}"
+                acme_email="${cfg.k8s.charts.acme_email}"
+                grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)"
+                grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)"
+            )
+
+            # -------------------- no modifications below --------------------
+            EOF
+            cat << '"'"'EOF'"'"'
+            apply=''${KUBECTL_CMD:-apply}
+
+            substitute_all () {
+                local x i k v subs
+                x="$(</dev/stdin)"
+                subs=("$@")
+                for i in "''${subs[@]}"; do
+                    k=$(echo "$i" | cut -d= -f1)
+                    v=$(echo "$i" | cut -d= -f2 | sed '"'"'s,/,\\/,g'"'"')
+                    x=$(echo "$x" | sed "s/@$k@/$v/g")
+                done
+                echo "$x"
+            }
+
+            substitute_defaults () {
+              substitute_all "''${vars[@]}"
+            }
+
+            create_namespace () {
+              kubectl get ns $namespace >/dev/null 2>&1 || kubectl create ns $namespace
+            }
+
+            export_vars () {
+              local i
+              for i in "''${vars[@]}"; do eval "$i"; done
+            }
+
+            kubectl_apply () {
+                local x
+                x="$(</dev/stdin)"
+                create_namespace
+                echo "$x" | \
+                  substitute_defaults | \
+                  kubectl -n $namespace $apply $@ -f -
+            }
+
+            kubectl_apply_files () {
+                local i charts
+                charts=("$@")
+                for i in "''${charts[@]}"; do
+                   cat "$i" | kubectl_apply
+                done
+            }
+
+            helm_apply () {
+                local name chart
+                name="$1"; shift
+                chart="$1"; shift
+                create_namespace $namespace
+                cat values.yaml | \
+                  substitute_defaults | \
+                  helm template -n $namespace -f- $name $chart $@ | \
+                  kubectl -n $namespace $apply -f -
+            }
+
+            [ -z "$namespace" ] && echo "namespace must be set" && exit 1
+
+            if [ "$apply" = delete ]; then
+                echo "WARNING: Really delete $(basename $TOP)? YES/no:"
+                read ans
+                [ "$ans" != YES ] && exit 0
+            fi
+
+            export_vars
+            EOF
+        '';
+        copy-kubernetes-charts = ''
+          #!/usr/bin/env bash
+          if [ \$# = 0 ]; then
+            dest=${cfg.clusterName}-charts
+          else
+            dest=\$1
+          fi
+          [ -e \$dest.bak ] && rm -rf \$dest.bak
+          [ -e \$dest ] && mv \$dest \$dest.bak
+          echo "Copying charts into \$dest"
+          cp -r ${kubernetes-charts}/share/kubernetes-charts \$dest
+          chown -R \$USER \$dest
+          chmod -R u+w \$dest
+          $out/bin/show-kubernetes-charts-config > \$dest/config.sh
+        '';
+      in
+      pkgs.stdenv.mkDerivation {
+        name = "cluster-scripts";
+        src = ../scripts;
+        buildCommand = ''
+          mkdir -p $out/bin
+          cp $src/* $out/bin
+          echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config
+          chmod a+x $out/bin/show-kubernetes-charts-config
+          echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts
+          chmod a+x $out/bin/copy-kubernetes-charts
+        '';
+  };
+
+  kubernetes-charts = pkgs.stdenv.mkDerivation rec {
+      name = "kubernetes-charts";
+      src = ../charts;
+      buildCommand = ''
+        mkdir -p $out/share/${name}
+        cp -r $src/* $out/share/${name}
+      '';
+  };
+
+  install-apitoken = ''
+      #!${pkgs.bash}/bin/bash
+      set -e
+      if [ -d /var/lib/cfssl ]; then
+        cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
+        chown cfssl /var/lib/cfssl/apitoken.secret
+        chmod 640 /var/lib/cfssl/apitoken.secret
+      else
+        mkdir -p /var/lib/kubernetes/secrets
+        cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
+        chown root /var/lib/kubernetes/secrets/apitoken.secret
+        chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
+      fi
+    '';
+
+  kubeMaster = {
+    services.cfssl.ca = pki.ca.cert;
+    services.cfssl.caKey = pki.ca.key;
+    services.kubernetes = {
+      roles = [ "master" ];
+      inherit apiserverAddress;
+      masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
+      clusterCidr = cfg.k8s.cidr;
+      pki.genCfsslCACert = false;
+      pki.genCfsslAPIToken = false;
+      pki.caCertPathPrefix = "${pki.initca}/ca";
+
+      kubelet = {
+        # clusterDomain = "${cfg.clusterName}.local";
+      };
+
+      apiserver = {
+        advertiseAddress = masterAddress;
+        authorizationMode = [ "Node" "RBAC" ];
+        allowPrivileged = true;
+        securePort = 4443;
+        insecurePort = 8080;
+        extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
+        extraSANs = cfg.k8s.master.extraSANs;
+        # verbosity = 4;
+      };
+
+      controllerManager = {
+        bindAddress = masterAddress;
+        extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
+      };
+
+      scheduler.address = masterAddress;
+
+      addonManager.enable = true;
+      addons = {
+        dns = {
+          enable = true;
+          # clusterDomain = "${cfg.clusterName}.local";
+          reconcileMode = "EnsureExists";
+        };
+      };
+    };
+
+    services.etcd = {
+      listenClientUrls = [ "https://${masterAddress}:2379" ];
+    };
+
+    networking.firewall = {
+      allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
+      allowedUDPPorts = [ 53 4053 ];
+    };
+
+    environment.systemPackages = [
+      pkgs.kubernetes-helm
+      pkgs.kubectl
+      cluster-scripts
+      kubernetes-charts
+    ];
+
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "cfssl.service" ];
+      before = [ "cfssl.target" ];
+      script = install-apitoken;
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
+
+    # systemd.services.cfssl-restart  = {
+    #   enable = true;
+    #   startAt  = "00/6:00";
+    #   description = "Restrart cfssl which regularly locks up";
+    #   script = "systemctl restart cfssl.service";
+    # };
+
+    systemd.services.kube-socat-https-proxy  = {
+      enable = true;
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kubelet.target" ];
+      description = "Proxy TCP port 443 to ingress NodePort at 30443";
+      script = "${pkgs.socat}/bin/socat TCP-LISTEN:443,fork,reuseaddr TCP:127.0.0.1:30443";
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
+  };
+
+  kubeNode = {
+    services.kubernetes = rec {
+      roles = [ "node" ];
+      inherit apiserverAddress;
+      # masterAddress = cfg.k8s.master.name;
+      masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
+      clusterCidr = cfg.k8s.cidr;
+      # kubelet.clusterDomain = "${cfg.clusterName}.local";
+      kubelet.hostname = "${cfg.hostName}.${cfg.domain}";
+      proxy.hostname = "${cfg.hostName}.${cfg.domain}";
+    };
+
+    networking = {
+      firewall = {
+        enable = true;
+        allowedTCPPorts = [ 4194 10250 ];
+        allowedUDPPorts = [ 53 ];
+      };
+    };
+    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
+    virtualisation.docker.autoPrune.enable = true;
+    systemd.services.kube-certmgr-apitoken-bootstrap = {
+      description = "Kubernetes certmgr bootstrapper";
+      wantedBy = [ "certmgr.service" ];
+      before = [ "certmgr.service" ];
+      script = install-apitoken;
+      serviceConfig = {
+        RestartSec = "10s";
+        Restart = "on-failure";
+      };
+    };
+  };
+
+in {
+  options.cluster.k8s = {
+    enable = mkEnableOption "Enable kubernetes";
+    nodes = mkOption {
+      type = types.listOf types.attrs;
+      default = [];
+    };
+
+    fileserver = mkOption {
+      type = types.str;
+      default = null;
+    };
+
+    cidr = mkOption {
+      type = types.str;
+      default = "10.0.0.0/16";
+    };
+
+    ingressNodes = mkOption {
+      type = types.listOf types.str;
+      default = null;
+    };
+
+    master = {
+      enable = mkEnableOption "Enable kubernetes master node";
+
+      address = mkOption {
+        type = types.str;
+        default = null;
+      };
+
+      name = mkOption {
+        type = types.str;
+        default = null;
+      };
+
+      extraSANs = mkOption {
+        type = types.listOf types.str;
+        default = [];
+      };
+
+      hw = mkOption {
+        type = types.path;
+        default = null;
+      };
+    };
+
+    node = {
+      enable = mkEnableOption "Enable kubernetes";
+    };
+
+    charts = {
+      acme_email = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_smtp_user = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      grafana_smtp_password = mkOption {
+        type = types.str;
+        default = "";
+      };
+    };
+  };
+
+  config = mkIf cfg.k8s.enable (
+    mkMerge [
+      (mkIf cfg.k8s.master.enable kubeMaster)
+      (mkIf cfg.k8s.node.enable kubeNode)
+    ]
+  );
+
+  imports = [ ./os.nix ];
+
+}
+

modules.bak/modules.nix

@@ -0,0 +1,7 @@
+{
+  imports = [
+    ./os.nix
+    ./fs.nix
+    ./k8s.nix
+  ];
+}

modules.bak/os.nix

@@ -0,0 +1,118 @@
+{ pkgs, lib, config, ... }:
+with lib;
+let
+  cfg = config.cluster;
+
+  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
+
+  common = {
+    users.extraUsers.admin.openssh.authorizedKeys.keys =
+      cfg.adminAuthorizedKeys;
+
+    users.extraUsers.root.openssh.authorizedKeys.keys =
+      cfg.adminAuthorizedKeys;
+
+    networking = {
+      domain = cfg.domain;
+      search = cfg.searchDomains;
+      extraHosts = cfg.extraHosts;
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 80 443 111 ];
+      firewall.allowedUDPPorts = [ 111 24007 24008 ];
+    } // (
+      if cfg.externalInterface == null then
+        { hostName = cfg.hostName; }
+      else {
+        hostName = cfg.hostName;
+        interfaces."${cfg.externalInterface}" = {
+          useDHCP = false;
+          ipv4.addresses = [ {
+            address = cfg.address;
+            prefixLength = 24;
+          } ];
+        };
+        defaultGateway = cfg.defaultGateway;
+        nameservers = cfg.nameservers;
+      }
+    );
+
+    security.pki.certificateFiles = [ pki.ca.cert ];
+    boot.kernel.sysctl = {
+      "kernel.mm.transparent_hugepage.enabled" = "never";
+      "net.core.somaxconn" = "512";
+    };
+
+    environment.systemPackages = with pkgs; [
+      nfs-utils
+    ];
+  };
+in
+{
+  options.cluster = {
+    initca = mkOption {
+      type = types.path;
+    };
+
+    hostName = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+    };
+
+    address = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+    };
+
+    externalInterface = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+    };
+
+
+    defaultGateway = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+    };
+
+    nameservers = mkOption {
+      type = types.listOf types.str;
+      default = [ "8.8.8.8" ];
+    };
+
+    domain = mkOption {
+      type = types.str;
+      default = null;
+    };
+
+    searchDomains = mkOption {
+      type = types.listOf types.str;
+      default = [ cfg.domain ];
+    };
+
+    cert = mkOption {
+      type = types.attrs;
+      default = null;
+    };
+
+    clusterName = mkOption {
+      type = types.str;
+      default = null;
+    };
+
+    extraHosts = mkOption {
+      type = types.str;
+    };
+
+    adminAuthorizedKeys = mkOption {
+      type = types.listOf types.str;
+      default = [];
+    };
+  };
+
+  config = common;
+
+  imports = [
+    ../nixos/configuration.nix
+  ];
+
+}

modules.bak/pki.nix

@@ -0,0 +1,82 @@
+{ pkgs, ca ? "", algo ? "rsa" }:
+let
+  initca = import ./initca.nix { inherit pkgs ca; };
+
+  ca' = {
+    key = "${initca}/ca-key.pem";
+    cert = "${initca}/ca.pem";
+  };
+
+  ca-config = pkgs.writeText "ca-config.json" ''
+  {
+      "signing": {
+          "default": {
+              "expiry": "8760h"
+          },
+          "profiles": {
+              "default": {
+                  "usages": [
+                      "signing",
+                      "key encipherment",
+                      "server auth",
+                      "client auth"
+                  ],
+                  "expiry": "8760h"
+              }
+          }
+      }
+  }
+  '';
+
+  gencsr = args:
+    let
+      csr = {
+        CN = "${args.cn}";
+        key = {
+          inherit algo;
+          size = if algo == "ecdsa" then 256 else 2048;
+        };
+        names = [
+          {
+            CN = "${args.cn}";
+            O = "${args.o}";
+            OU = "${args.cn}.${args.o}.pki.caSpec";
+            L = "generated";
+          }
+        ];
+        hosts = args.hosts;
+      };
+    in
+      pkgs.writeText "${args.cn}-csr.json" (builtins.toJSON csr);
+in
+# Example usage:
+#
+# gencert { cn = "test"; ca = ca; o = "test; };
+#
+rec {
+  inherit initca;
+  ca = ca';
+  gencert = attrs:
+    let
+      conf = {
+        cn = attrs.cn;
+        ca = attrs.ca;
+        csr = gencsr { cn = attrs.cn; o = attrs.o; hosts = attrs.hosts; };
+      };
+      cfssl = conf:
+        ''
+          cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
+          -config=${ca-config} -profile=default ${conf.csr} | \
+          cfssljson -bare cert; \
+          mkdir -p $out; cp *.pem $out
+        '';
+      crt =
+        pkgs.runCommand "${attrs.cn}" {
+          buildInputs = [ pkgs.cfssl ];
+        } (cfssl conf);
+    in
+    {
+      key = "${crt}/cert-key.pem";
+      cert = "${crt}/cert.pem";
+    };
+}