Move modules and overrides to submodule

modules.bak/default.nix

@@ -1,130 +0,0 @@
-{ pkgs, cluster, customize ? {}, extraConfig ? {}, lib, config, ... }:
-with lib;
-let
-  cfg = cluster;
-
-  mkSANs = host: [
-    host.name
-    host.address
-    "127.0.0.1"
-  ];
-
-  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
-
-  mkCert = host: {
-    ${host.name} = pki.gencert {
-      cn = host.name;
-      ca = cfg.ca;
-      o = cfg.clusterName;
-      hosts = [
-        host.name
-        "${host.name}.${cfg.domain}"
-        host.address
-        "127.0.0.1"
-      ];
-    };
-  };
-in {
-  k8s = rec {
-    apiserver = host: self: {
-      deployment.targetHost = host.address;
-
-      inherit customize;
-
-      cluster = mkMerge [
-        cfg
-        {
-          hostName = host.name;
-          address = host.address;
-          k8s.master.enable = true;
-          k8s.node.enable = true;
-        }
-      ];
-
-      services.kubernetes.kubelet.extraSANs = mkSANs host;
-
-      imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
-    };
-
-    node = host: self: {
-      deployment.targetHost = host.address;
-
-      inherit customize;
-
-      cluster = mkMerge [
-        cfg
-        {
-          hostName = host.name;
-          address = host.address;
-          k8s.node.enable = true;
-        }
-      ];
-
-      services.kubernetes.kubelet.extraSANs = mkSANs host;
-
-      imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
-    };
-
-    mkDeployment = top:
-      let
-        master = cfg.k8s.master // { hw = "${top}/${master.name}.nix"; };
-        nodes = cfg.k8s.nodes;
-        server = { "${master.name}" = apiserver master; };
-      in
-        builtins.foldl' (a: x:
-          a // {
-            "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
-          }) server nodes;
-  };
-
-  fs = rec {
-    node = host: self: {
-      deployment.targetHost = host.address;
-
-      inherit customize;
-
-      cluster = mkMerge [
-        { fs.nfs.enable = mkDefault true; }
-        cfg
-        {
-          hostName = host.name;
-          address = host.address;
-          cert = mkCert host.name;
-        }
-      ];
-
-      imports = [ host.hw extraConfig ./modules.nix ];
-    };
-
-    mkDeployment = top: nodes:
-      builtins.foldl' (a: x:
-        a // {
-          "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
-        }) {} nodes;
-  } ;
-
-  host = rec {
-    node = host: self: {
-      deployment.targetHost = host.address;
-
-      inherit customize;
-
-      cluster = mkMerge [
-        cfg
-        {
-          hostName = host.name;
-          address = host.address;
-          cert = mkCert host.name;
-        }
-      ];
-
-      imports = [ host.hw extraConfig ./modules.nix ];
-    };
-
-    mkDeployment = top: nodes:
-      builtins.foldl' (a: x:
-        a // {
-          "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
-        }) {} nodes;
-    };
-}

modules.bak/fs.nix

@@ -1,67 +0,0 @@
-{ pkgs, lib, config, ... }:
-with lib;
-let
-  cfg = config.cluster.fs;
-
-  cert = cfg.cert;
-
-  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
-
-  common = {
-    boot.kernelModules = [
-      "dm_snapshot"
-      "dm_mirror"
-      "dm_thin_pool"
-    ];
-
-    networking = {
-      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
-      firewall.allowedTCPPorts = [ 111 2049 ];
-      firewall.allowedUDPPorts = [ 111 2049 24007 24008 ];
-    };
-
-    environment.systemPackages = [ pkgs.lvm2 ];
-  };
-
-  glusterfs = {
-    services.glusterfs = {
-      enable = true;
-      tlsSettings = {
-        caCert = pki.ca.cert;
-        tlsKeyPath = cert.key;
-        tlsPem = cert.cert;
-      };
-    };
-  };
-
-  nfs = {
-    services.nfs.server = {
-      enable = true;
-      exports = cfg.nfs.exports;
-    };
-  };
-in {
-  options.cluster.fs = {
-    enable = mkEnableOption "Enable nfs fileserver";
-
-    nfs = {
-      enable = mkEnableOption "Enable nfs fileserver";
-      exports = mkOption {
-        type = types.str;
-        default = "";
-      };
-    };
-
-    glusterfs.enable = mkEnableOption "Enable glusterfs fileserver";
-  };
-
-  config = mkIf cfg.enable (
-    mkMerge [
-      common
-      (mkIf cfg.nfs.enable nfs)
-      (mkIf cfg.glusterfs.enable glusterfs)
-    ]
-  );
-
-  imports = [ ./os.nix ];
-}

modules.bak/initca.nix

@@ -1,42 +0,0 @@
-{
-  pkgs ? import <nixpkgs> {},
-  ca ? null,
-  name ? "ca",
-  algo ? "rsa",
-  hosts ? [],
-  ...}:
-with pkgs;
-let
-  ca_csr = pkgs.writeText "${name}-csr.json" (builtins.toJSON {
-      inherit hosts;
-      CN = "${name}";
-      key = {
-        inherit algo;
-        size = if algo == "ecdsa" then 256 else 2048;
-      };
-      names = [
-        {
-          CN = "${name}";
-          O = "NixOS";
-          OU = "${name}.pki.caSpec";
-          L = "generated";
-        }
-      ];
-    }
-  );
-  ca' =
-      pkgs.runCommand "initca" {
-        buildInputs = [ pkgs.cfssl ];
-      } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca;
-           mkdir -p $out; cp *.pem $out '';
-   initca = if ca != null then ca else ca';
-in
-  # make ca derivation sha depend on initca cfssl output
-  pkgs.stdenv.mkDerivation {
-    inherit name;
-    src = initca;
-    buildCommand = ''
-      mkdir -p $out;
-      cp -r $src/* $out
-    '';
-  }

modules.bak/k8s.nix

@@ -1,384 +0,0 @@
-{ pkgs, lib, config, ...}:
-with lib;
-let
-  cfg = config.cluster;
-
-  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
-
-  masterAddress = cfg.k8s.master.address;
-
-  apiserverAddress = "https://${masterAddress}:4443";
-
-  cfssl-apitoken =
-    let
-      apitoken = pkgs.stdenv.mkDerivation {
-        name = "apitoken";
-        buildCommand = ''
-          head -c ${toString (32 / 2)} /dev/urandom | \
-          od -An -t x | tr -d ' ' > $out
-          chmod 400 $out
-        '';
-      };
-    in
-    # make ca derivation sha depend on initca cfssl output
-    pkgs.stdenv.mkDerivation {
-      name = "cfssl-apitoken";
-      src = apitoken;
-      buildCommand = ''
-        cp $src $out
-      '';
-    };
-
-    cluster-scripts =
-      let
-        first = builtins.head cfg.k8s.ingressNodes;
-        rest =  builtins.tail cfg.k8s.ingressNodes;
-        ingressNodes = builtins.foldl' (a: x:
-          a + ",${x}") first rest;
-        ingressReplicaCount =
-           builtins.toString (builtins.length cfg.k8s.ingressNodes);
-        show-kubernetes-charts-config = ''
-            #!/usr/bin/env bash
-            cat << EOF
-            # Generated by show-kubernetes-charts-config
-            #   $(date)
-            # Charts in ${kubernetes-charts}/share/kubernetes-charts
-
-            vars=(
-                initca="${pki.initca}"
-                apiserver="${cfg.k8s.master.name}"
-                cluster="${cfg.clusterName}"
-                ingress_nodes="${ingressNodes}"
-                ingress_replica_count="${ingressReplicaCount}"
-                fileserver="${cfg.k8s.fileserver}"
-                acme_email="${cfg.k8s.charts.acme_email}"
-                grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)"
-                grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)"
-            )
-
-            # -------------------- no modifications below --------------------
-            EOF
-            cat << '"'"'EOF'"'"'
-            apply=''${KUBECTL_CMD:-apply}
-
-            substitute_all () {
-                local x i k v subs
-                x="$(</dev/stdin)"
-                subs=("$@")
-                for i in "''${subs[@]}"; do
-                    k=$(echo "$i" | cut -d= -f1)
-                    v=$(echo "$i" | cut -d= -f2 | sed '"'"'s,/,\\/,g'"'"')
-                    x=$(echo "$x" | sed "s/@$k@/$v/g")
-                done
-                echo "$x"
-            }
-
-            substitute_defaults () {
-              substitute_all "''${vars[@]}"
-            }
-
-            create_namespace () {
-              kubectl get ns $namespace >/dev/null 2>&1 || kubectl create ns $namespace
-            }
-
-            export_vars () {
-              local i
-              for i in "''${vars[@]}"; do eval "$i"; done
-            }
-
-            kubectl_apply () {
-                local x
-                x="$(</dev/stdin)"
-                create_namespace
-                echo "$x" | \
-                  substitute_defaults | \
-                  kubectl -n $namespace $apply $@ -f -
-            }
-
-            kubectl_apply_files () {
-                local i charts
-                charts=("$@")
-                for i in "''${charts[@]}"; do
-                   cat "$i" | kubectl_apply
-                done
-            }
-
-            helm_apply () {
-                local name chart
-                name="$1"; shift
-                chart="$1"; shift
-                create_namespace $namespace
-                cat values.yaml | \
-                  substitute_defaults | \
-                  helm template -n $namespace -f- $name $chart $@ | \
-                  kubectl -n $namespace $apply -f -
-            }
-
-            [ -z "$namespace" ] && echo "namespace must be set" && exit 1
-
-            if [ "$apply" = delete ]; then
-                echo "WARNING: Really delete $(basename $TOP)? YES/no:"
-                read ans
-                [ "$ans" != YES ] && exit 0
-            fi
-
-            export_vars
-            EOF
-        '';
-        copy-kubernetes-charts = ''
-          #!/usr/bin/env bash
-          if [ \$# = 0 ]; then
-            dest=${cfg.clusterName}-charts
-          else
-            dest=\$1
-          fi
-          [ -e \$dest.bak ] && rm -rf \$dest.bak
-          [ -e \$dest ] && mv \$dest \$dest.bak
-          echo "Copying charts into \$dest"
-          cp -r ${kubernetes-charts}/share/kubernetes-charts \$dest
-          chown -R \$USER \$dest
-          chmod -R u+w \$dest
-          $out/bin/show-kubernetes-charts-config > \$dest/config.sh
-        '';
-      in
-      pkgs.stdenv.mkDerivation {
-        name = "cluster-scripts";
-        src = ../scripts;
-        buildCommand = ''
-          mkdir -p $out/bin
-          cp $src/* $out/bin
-          echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config
-          chmod a+x $out/bin/show-kubernetes-charts-config
-          echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts
-          chmod a+x $out/bin/copy-kubernetes-charts
-        '';
-  };
-
-  kubernetes-charts = pkgs.stdenv.mkDerivation rec {
-      name = "kubernetes-charts";
-      src = ../charts;
-      buildCommand = ''
-        mkdir -p $out/share/${name}
-        cp -r $src/* $out/share/${name}
-      '';
-  };
-
-  install-apitoken = ''
-      #!${pkgs.bash}/bin/bash
-      set -e
-      if [ -d /var/lib/cfssl ]; then
-        cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
-        chown cfssl /var/lib/cfssl/apitoken.secret
-        chmod 640 /var/lib/cfssl/apitoken.secret
-      else
-        mkdir -p /var/lib/kubernetes/secrets
-        cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
-        chown root /var/lib/kubernetes/secrets/apitoken.secret
-        chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
-      fi
-    '';
-
-  kubeMaster = {
-    services.cfssl.ca = pki.ca.cert;
-    services.cfssl.caKey = pki.ca.key;
-    services.kubernetes = {
-      roles = [ "master" ];
-      inherit apiserverAddress;
-      masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
-      clusterCidr = cfg.k8s.cidr;
-      pki.genCfsslCACert = false;
-      pki.genCfsslAPIToken = false;
-      pki.caCertPathPrefix = "${pki.initca}/ca";
-
-      kubelet = {
-        # clusterDomain = "${cfg.clusterName}.local";
-      };
-
-      apiserver = {
-        advertiseAddress = masterAddress;
-        authorizationMode = [ "Node" "RBAC" ];
-        allowPrivileged = true;
-        securePort = 4443;
-        insecurePort = 8080;
-        extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
-        extraSANs = cfg.k8s.master.extraSANs;
-        # verbosity = 4;
-      };
-
-      controllerManager = {
-        bindAddress = masterAddress;
-        extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
-      };
-
-      scheduler.address = masterAddress;
-
-      addonManager.enable = true;
-      addons = {
-        dns = {
-          enable = true;
-          # clusterDomain = "${cfg.clusterName}.local";
-          reconcileMode = "EnsureExists";
-        };
-      };
-    };
-
-    services.etcd = {
-      listenClientUrls = [ "https://${masterAddress}:2379" ];
-    };
-
-    networking.firewall = {
-      allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
-      allowedUDPPorts = [ 53 4053 ];
-    };
-
-    environment.systemPackages = [
-      pkgs.kubernetes-helm
-      pkgs.kubectl
-      cluster-scripts
-      kubernetes-charts
-    ];
-
-    systemd.services.kube-certmgr-apitoken-bootstrap = {
-      description = "Kubernetes certmgr bootstrapper";
-      wantedBy = [ "cfssl.service" ];
-      before = [ "cfssl.target" ];
-      script = install-apitoken;
-      serviceConfig = {
-        RestartSec = "10s";
-        Restart = "on-failure";
-      };
-    };
-
-    # systemd.services.cfssl-restart  = {
-    #   enable = true;
-    #   startAt  = "00/6:00";
-    #   description = "Restrart cfssl which regularly locks up";
-    #   script = "systemctl restart cfssl.service";
-    # };
-
-    systemd.services.kube-socat-https-proxy  = {
-      enable = true;
-      wantedBy = [ "kubernetes.target" ];
-      after = [ "kubelet.target" ];
-      description = "Proxy TCP port 443 to ingress NodePort at 30443";
-      script = "${pkgs.socat}/bin/socat TCP-LISTEN:443,fork,reuseaddr TCP:127.0.0.1:30443";
-      serviceConfig = {
-        RestartSec = "10s";
-        Restart = "on-failure";
-      };
-    };
-  };
-
-  kubeNode = {
-    services.kubernetes = rec {
-      roles = [ "node" ];
-      inherit apiserverAddress;
-      # masterAddress = cfg.k8s.master.name;
-      masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
-      clusterCidr = cfg.k8s.cidr;
-      # kubelet.clusterDomain = "${cfg.clusterName}.local";
-      kubelet.hostname = "${cfg.hostName}.${cfg.domain}";
-      proxy.hostname = "${cfg.hostName}.${cfg.domain}";
-    };
-
-    networking = {
-      firewall = {
-        enable = true;
-        allowedTCPPorts = [ 4194 10250 ];
-        allowedUDPPorts = [ 53 ];
-      };
-    };
-    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
-    virtualisation.docker.autoPrune.enable = true;
-    systemd.services.kube-certmgr-apitoken-bootstrap = {
-      description = "Kubernetes certmgr bootstrapper";
-      wantedBy = [ "certmgr.service" ];
-      before = [ "certmgr.service" ];
-      script = install-apitoken;
-      serviceConfig = {
-        RestartSec = "10s";
-        Restart = "on-failure";
-      };
-    };
-  };
-
-in {
-  options.cluster.k8s = {
-    enable = mkEnableOption "Enable kubernetes";
-    nodes = mkOption {
-      type = types.listOf types.attrs;
-      default = [];
-    };
-
-    fileserver = mkOption {
-      type = types.str;
-      default = null;
-    };
-
-    cidr = mkOption {
-      type = types.str;
-      default = "10.0.0.0/16";
-    };
-
-    ingressNodes = mkOption {
-      type = types.listOf types.str;
-      default = null;
-    };
-
-    master = {
-      enable = mkEnableOption "Enable kubernetes master node";
-
-      address = mkOption {
-        type = types.str;
-        default = null;
-      };
-
-      name = mkOption {
-        type = types.str;
-        default = null;
-      };
-
-      extraSANs = mkOption {
-        type = types.listOf types.str;
-        default = [];
-      };
-
-      hw = mkOption {
-        type = types.path;
-        default = null;
-      };
-    };
-
-    node = {
-      enable = mkEnableOption "Enable kubernetes";
-    };
-
-    charts = {
-      acme_email = mkOption {
-        type = types.str;
-        default = "";
-      };
-
-      grafana_smtp_user = mkOption {
-        type = types.str;
-        default = "";
-      };
-
-      grafana_smtp_password = mkOption {
-        type = types.str;
-        default = "";
-      };
-    };
-  };
-
-  config = mkIf cfg.k8s.enable (
-    mkMerge [
-      (mkIf cfg.k8s.master.enable kubeMaster)
-      (mkIf cfg.k8s.node.enable kubeNode)
-    ]
-  );
-
-  imports = [ ./os.nix ];
-
-}
-

modules.bak/modules.nix

@@ -1,7 +0,0 @@
-{
-  imports = [
-    ./os.nix
-    ./fs.nix
-    ./k8s.nix
-  ];
-}

modules.bak/os.nix

@@ -1,118 +0,0 @@
-{ pkgs, lib, config, ... }:
-with lib;
-let
-  cfg = config.cluster;
-
-  pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
-
-  common = {
-    users.extraUsers.admin.openssh.authorizedKeys.keys =
-      cfg.adminAuthorizedKeys;
-
-    users.extraUsers.root.openssh.authorizedKeys.keys =
-      cfg.adminAuthorizedKeys;
-
-    networking = {
-      domain = cfg.domain;
-      search = cfg.searchDomains;
-      extraHosts = cfg.extraHosts;
-      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
-      firewall.allowedTCPPorts = [ 80 443 111 ];
-      firewall.allowedUDPPorts = [ 111 24007 24008 ];
-    } // (
-      if cfg.externalInterface == null then
-        { hostName = cfg.hostName; }
-      else {
-        hostName = cfg.hostName;
-        interfaces."${cfg.externalInterface}" = {
-          useDHCP = false;
-          ipv4.addresses = [ {
-            address = cfg.address;
-            prefixLength = 24;
-          } ];
-        };
-        defaultGateway = cfg.defaultGateway;
-        nameservers = cfg.nameservers;
-      }
-    );
-
-    security.pki.certificateFiles = [ pki.ca.cert ];
-    boot.kernel.sysctl = {
-      "kernel.mm.transparent_hugepage.enabled" = "never";
-      "net.core.somaxconn" = "512";
-    };
-
-    environment.systemPackages = with pkgs; [
-      nfs-utils
-    ];
-  };
-in
-{
-  options.cluster = {
-    initca = mkOption {
-      type = types.path;
-    };
-
-    hostName = mkOption {
-      type = types.nullOr types.str;
-      default = null;
-    };
-
-    address = mkOption {
-      type = types.nullOr types.str;
-      default = null;
-    };
-
-    externalInterface = mkOption {
-      type = types.nullOr types.str;
-      default = null;
-    };
-
-
-    defaultGateway = mkOption {
-      type = types.nullOr types.str;
-      default = null;
-    };
-
-    nameservers = mkOption {
-      type = types.listOf types.str;
-      default = [ "8.8.8.8" ];
-    };
-
-    domain = mkOption {
-      type = types.str;
-      default = null;
-    };
-
-    searchDomains = mkOption {
-      type = types.listOf types.str;
-      default = [ cfg.domain ];
-    };
-
-    cert = mkOption {
-      type = types.attrs;
-      default = null;
-    };
-
-    clusterName = mkOption {
-      type = types.str;
-      default = null;
-    };
-
-    extraHosts = mkOption {
-      type = types.str;
-    };
-
-    adminAuthorizedKeys = mkOption {
-      type = types.listOf types.str;
-      default = [];
-    };
-  };
-
-  config = common;
-
-  imports = [
-    ../nixos/configuration.nix
-  ];
-
-}

modules.bak/pki.nix

@@ -1,82 +0,0 @@
-{ pkgs, ca ? "", algo ? "rsa" }:
-let
-  initca = import ./initca.nix { inherit pkgs ca; };
-
-  ca' = {
-    key = "${initca}/ca-key.pem";
-    cert = "${initca}/ca.pem";
-  };
-
-  ca-config = pkgs.writeText "ca-config.json" ''
-  {
-      "signing": {
-          "default": {
-              "expiry": "8760h"
-          },
-          "profiles": {
-              "default": {
-                  "usages": [
-                      "signing",
-                      "key encipherment",
-                      "server auth",
-                      "client auth"
-                  ],
-                  "expiry": "8760h"
-              }
-          }
-      }
-  }
-  '';
-
-  gencsr = args:
-    let
-      csr = {
-        CN = "${args.cn}";
-        key = {
-          inherit algo;
-          size = if algo == "ecdsa" then 256 else 2048;
-        };
-        names = [
-          {
-            CN = "${args.cn}";
-            O = "${args.o}";
-            OU = "${args.cn}.${args.o}.pki.caSpec";
-            L = "generated";
-          }
-        ];
-        hosts = args.hosts;
-      };
-    in
-      pkgs.writeText "${args.cn}-csr.json" (builtins.toJSON csr);
-in
-# Example usage:
-#
-# gencert { cn = "test"; ca = ca; o = "test; };
-#
-rec {
-  inherit initca;
-  ca = ca';
-  gencert = attrs:
-    let
-      conf = {
-        cn = attrs.cn;
-        ca = attrs.ca;
-        csr = gencsr { cn = attrs.cn; o = attrs.o; hosts = attrs.hosts; };
-      };
-      cfssl = conf:
-        ''
-          cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
-          -config=${ca-config} -profile=default ${conf.csr} | \
-          cfssljson -bare cert; \
-          mkdir -p $out; cp *.pem $out
-        '';
-      crt =
-        pkgs.runCommand "${attrs.cn}" {
-          buildInputs = [ pkgs.cfssl ];
-        } (cfssl conf);
-    in
-    {
-      key = "${crt}/cert-key.pem";
-      cert = "${crt}/cert.pem";
-    };
-}

overrides/kubelet.nix

@@ -1,359 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  top = config.services.kubernetes;
-  cfg = top.kubelet;
-
-  cniConfig =
-    if cfg.cni.config != [] && cfg.cni.configDir != null then
-      throw "Verbatim CNI-config and CNI configDir cannot both be set."
-    else if cfg.cni.configDir != null then
-      cfg.cni.configDir
-    else
-      (pkgs.buildEnv {
-        name = "kubernetes-cni-config";
-        paths = imap (i: entry:
-          pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
-        ) cfg.cni.config;
-      });
-
-  infraContainer = pkgs.dockerTools.buildImage {
-    name = "pause";
-    tag = "latest";
-    contents = top.package.pause;
-    config.Cmd = "/bin/pause";
-  };
-
-  kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
-
-  manifestPath = "kubernetes/manifests";
-
-  taintOptions = with lib.types; { name, ... }: {
-    options = {
-      key = mkOption {
-        description = "Key of taint.";
-        default = name;
-        type = str;
-      };
-      value = mkOption {
-        description = "Value of taint.";
-        type = str;
-      };
-      effect = mkOption {
-        description = "Effect of taint.";
-        example = "NoSchedule";
-        type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
-      };
-    };
-  };
-
-  taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
-in
-{
-  disabledModules = [ "services/cluster/kubernetes/kubelet.nix" ];
-
-  imports = [
-    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
-    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
-    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
-  ];
-
-  ###### interface
-  options.services.kubernetes.kubelet = with lib.types; {
-
-    address = mkOption {
-      description = "Kubernetes kubelet info server listening address.";
-      default = "0.0.0.0";
-      type = str;
-    };
-
-    clusterDns = mkOption {
-      description = "Use alternative DNS.";
-      default = "10.1.0.1";
-      type = str;
-    };
-
-    clusterDomain = mkOption {
-      description = "Use alternative domain.";
-      default = config.services.kubernetes.addons.dns.clusterDomain;
-      type = str;
-    };
-
-    extraSANs = mkOption {
-      description = "Extra x509 Subject Alternative Names to be added to the kubelet tls cert.";
-      default = [];
-      type = listOf str;
-    };
-
-    clientCaFile = mkOption {
-      description = "Kubernetes apiserver CA file for client authentication.";
-      default = top.caFile;
-      type = nullOr path;
-    };
-
-    cni = {
-      packages = mkOption {
-        description = "List of network plugin packages to install.";
-        type = listOf package;
-        default = [];
-      };
-
-      config = mkOption {
-        description = "Kubernetes CNI configuration.";
-        type = listOf attrs;
-        default = [];
-        example = literalExample ''
-          [{
-            "cniVersion": "0.3.1",
-            "name": "mynet",
-            "type": "bridge",
-            "bridge": "cni0",
-            "isGateway": true,
-            "ipMasq": true,
-            "ipam": {
-                "type": "host-local",
-                "subnet": "10.22.0.0/16",
-                "routes": [
-                    { "dst": "0.0.0.0/0" }
-                ]
-            }
-          } {
-            "cniVersion": "0.3.1",
-            "type": "loopback"
-          }]
-        '';
-      };
-
-      configDir = mkOption {
-        description = "Path to Kubernetes CNI configuration directory.";
-        type = nullOr path;
-        default = null;
-      };
-    };
-
-    enable = mkEnableOption "Kubernetes kubelet.";
-
-    extraOpts = mkOption {
-      description = "Kubernetes kubelet extra command line options.";
-      default = "";
-      type = str;
-    };
-
-    featureGates = mkOption {
-      description = "List set of feature gates";
-      default = top.featureGates;
-      type = listOf str;
-    };
-
-    healthz = {
-      bind = mkOption {
-        description = "Kubernetes kubelet healthz listening address.";
-        default = "127.0.0.1";
-        type = str;
-      };
-
-      port = mkOption {
-        description = "Kubernetes kubelet healthz port.";
-        default = 10248;
-        type = int;
-      };
-    };
-
-    hostname = mkOption {
-      description = "Kubernetes kubelet hostname override.";
-      default = config.networking.hostName;
-      type = str;
-    };
-
-    kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
-
-    manifests = mkOption {
-      description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
-      type = attrsOf attrs;
-      default = {};
-    };
-
-    networkPlugin = mkOption {
-      description = "Network plugin to use by Kubernetes.";
-      type = nullOr (enum ["cni" "kubenet"]);
-      default = "kubenet";
-    };
-
-    nodeIp = mkOption {
-      description = "IP address of the node. If set, kubelet will use this IP address for the node.";
-      default = null;
-      type = nullOr str;
-    };
-
-    registerNode = mkOption {
-      description = "Whether to auto register kubelet with API server.";
-      default = true;
-      type = bool;
-    };
-
-    port = mkOption {
-      description = "Kubernetes kubelet info server listening port.";
-      default = 10250;
-      type = int;
-    };
-
-    seedDockerImages = mkOption {
-      description = "List of docker images to preload on system";
-      default = [];
-      type = listOf package;
-    };
-
-    taints = mkOption {
-      description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
-      default = {};
-      type = attrsOf (submodule [ taintOptions ]);
-    };
-
-    tlsCertFile = mkOption {
-      description = "File containing x509 Certificate for HTTPS.";
-      default = null;
-      type = nullOr path;
-    };
-
-    tlsKeyFile = mkOption {
-      description = "File containing x509 private key matching tlsCertFile.";
-      default = null;
-      type = nullOr path;
-    };
-
-    unschedulable = mkOption {
-      description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
-      default = false;
-      type = bool;
-    };
-
-    verbosity = mkOption {
-      description = ''
-        Optional glog verbosity level for logging statements. See
-        <link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
-      '';
-      default = null;
-      type = nullOr int;
-    };
-
-  };
-
-  ###### implementation
-  config = mkMerge [
-    (mkIf cfg.enable {
-      services.kubernetes.kubelet.seedDockerImages = [infraContainer];
-
-      systemd.services.kubelet = {
-        description = "Kubernetes Kubelet Service";
-        wantedBy = [ "kubernetes.target" ];
-        after = [ "network.target" "docker.service" "kube-apiserver.service" ];
-        path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
-        preStart = ''
-          ${concatMapStrings (img: ''
-            echo "Seeding docker image: ${img}"
-            docker load <${img}
-          '') cfg.seedDockerImages}
-
-          rm /opt/cni/bin/* || true
-          ${concatMapStrings (package: ''
-            echo "Linking cni package: ${package}"
-            ln -fs ${package}/bin/* /opt/cni/bin
-          '') cfg.cni.packages}
-        '';
-        serviceConfig = {
-          Slice = "kubernetes.slice";
-          CPUAccounting = true;
-          MemoryAccounting = true;
-          Restart = "on-failure";
-          RestartSec = "1000ms";
-          ExecStart = ''${top.package}/bin/kubelet \
-            --address=${cfg.address} \
-            --authentication-token-webhook \
-            --authentication-token-webhook-cache-ttl="10s" \
-            --authorization-mode=Webhook \
-            ${optionalString (cfg.clientCaFile != null)
-              "--client-ca-file=${cfg.clientCaFile}"} \
-            ${optionalString (cfg.clusterDns != "")
-              "--cluster-dns=${cfg.clusterDns}"} \
-            ${optionalString (cfg.clusterDomain != "")
-              "--cluster-domain=${cfg.clusterDomain}"} \
-            --cni-conf-dir=${cniConfig} \
-            ${optionalString (cfg.featureGates != [])
-              "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-            --hairpin-mode=hairpin-veth \
-            --healthz-bind-address=${cfg.healthz.bind} \
-            --healthz-port=${toString cfg.healthz.port} \
-            --hostname-override=${cfg.hostname} \
-            --kubeconfig=${kubeconfig} \
-            ${optionalString (cfg.networkPlugin != null)
-              "--network-plugin=${cfg.networkPlugin}"} \
-            ${optionalString (cfg.nodeIp != null)
-              "--node-ip=${cfg.nodeIp}"} \
-            --pod-infra-container-image=pause \
-            ${optionalString (cfg.manifests != {})
-              "--pod-manifest-path=/etc/${manifestPath}"} \
-            --port=${toString cfg.port} \
-            --register-node=${boolToString cfg.registerNode} \
-            ${optionalString (taints != "")
-              "--register-with-taints=${taints}"} \
-            --root-dir=${top.dataDir} \
-            ${optionalString (cfg.tlsCertFile != null)
-              "--tls-cert-file=${cfg.tlsCertFile}"} \
-            ${optionalString (cfg.tlsKeyFile != null)
-              "--tls-private-key-file=${cfg.tlsKeyFile}"} \
-            ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
-            ${cfg.extraOpts}
-          '';
-          WorkingDirectory = top.dataDir;
-        };
-      };
-
-      # Allways include cni plugins
-      services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
-
-      boot.kernelModules = ["br_netfilter"];
-
-      services.kubernetes.kubelet.hostname = with config.networking;
-        mkDefault (hostName + optionalString (domain != null) ".${domain}");
-
-      services.kubernetes.pki.certs = with top.lib; {
-        kubelet = mkCert {
-          name = "kubelet";
-          CN = top.kubelet.hostname;
-          hosts = top.kubelet.extraSANs;
-          action = "systemctl restart kubelet.service";
-
-        };
-        kubeletClient = mkCert {
-          name = "kubelet-client";
-          CN = "system:node:${top.kubelet.hostname}";
-          fields = {
-            O = "system:nodes";
-          };
-          action = "systemctl restart kubelet.service";
-        };
-      };
-
-      services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
-    })
-
-    (mkIf (cfg.enable && cfg.manifests != {}) {
-      environment.etc = mapAttrs' (name: manifest:
-        nameValuePair "${manifestPath}/${name}.json" {
-          text = builtins.toJSON manifest;
-          mode = "0755";
-        }
-      ) cfg.manifests;
-    })
-
-    (mkIf (cfg.unschedulable && cfg.enable) {
-      services.kubernetes.kubelet.taints.unschedulable = {
-        value = "true";
-        effect = "NoSchedule";
-      };
-    })
-
-  ];
-}