diff --git a/modules.bak/default.nix b/modules.bak/default.nix deleted file mode 100644 index 62a1f5c..0000000 --- a/modules.bak/default.nix +++ /dev/null @@ -1,130 +0,0 @@ -{ pkgs, cluster, customize ? {}, extraConfig ? {}, lib, config, ... }: -with lib; -let - cfg = cluster; - - mkSANs = host: [ - host.name - host.address - "127.0.0.1" - ]; - - pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; - - mkCert = host: { - ${host.name} = pki.gencert { - cn = host.name; - ca = cfg.ca; - o = cfg.clusterName; - hosts = [ - host.name - "${host.name}.${cfg.domain}" - host.address - "127.0.0.1" - ]; - }; - }; -in { - k8s = rec { - apiserver = host: self: { - deployment.targetHost = host.address; - - inherit customize; - - cluster = mkMerge [ - cfg - { - hostName = host.name; - address = host.address; - k8s.master.enable = true; - k8s.node.enable = true; - } - ]; - - services.kubernetes.kubelet.extraSANs = mkSANs host; - - imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ]; - }; - - node = host: self: { - deployment.targetHost = host.address; - - inherit customize; - - cluster = mkMerge [ - cfg - { - hostName = host.name; - address = host.address; - k8s.node.enable = true; - } - ]; - - services.kubernetes.kubelet.extraSANs = mkSANs host; - - imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ]; - }; - - mkDeployment = top: - let - master = cfg.k8s.master // { hw = "${top}/${master.name}.nix"; }; - nodes = cfg.k8s.nodes; - server = { "${master.name}" = apiserver master; }; - in - builtins.foldl' (a: x: - a // { - "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; }); - }) server nodes; - }; - - fs = rec { - node = host: self: { - deployment.targetHost = host.address; - - inherit customize; - - cluster = mkMerge [ - { fs.nfs.enable = mkDefault true; } - cfg - { - hostName = host.name; - address = host.address; - cert = mkCert host.name; - } - ]; - - imports = [ host.hw extraConfig ./modules.nix ]; - }; - - mkDeployment = top: nodes: - builtins.foldl' (a: x: - a // { - "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; }); - }) {} nodes; - } ; - - host = rec { - node = host: self: { - deployment.targetHost = host.address; - - inherit customize; - - cluster = mkMerge [ - cfg - { - hostName = host.name; - address = host.address; - cert = mkCert host.name; - } - ]; - - imports = [ host.hw extraConfig ./modules.nix ]; - }; - - mkDeployment = top: nodes: - builtins.foldl' (a: x: - a // { - "${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; }); - }) {} nodes; - }; -} diff --git a/modules.bak/fs.nix b/modules.bak/fs.nix deleted file mode 100644 index dec87b8..0000000 --- a/modules.bak/fs.nix +++ /dev/null @@ -1,67 +0,0 @@ -{ pkgs, lib, config, ... }: -with lib; -let - cfg = config.cluster.fs; - - cert = cfg.cert; - - pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; - - common = { - boot.kernelModules = [ - "dm_snapshot" - "dm_mirror" - "dm_thin_pool" - ]; - - networking = { - firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; - firewall.allowedTCPPorts = [ 111 2049 ]; - firewall.allowedUDPPorts = [ 111 2049 24007 24008 ]; - }; - - environment.systemPackages = [ pkgs.lvm2 ]; - }; - - glusterfs = { - services.glusterfs = { - enable = true; - tlsSettings = { - caCert = pki.ca.cert; - tlsKeyPath = cert.key; - tlsPem = cert.cert; - }; - }; - }; - - nfs = { - services.nfs.server = { - enable = true; - exports = cfg.nfs.exports; - }; - }; -in { - options.cluster.fs = { - enable = mkEnableOption "Enable nfs fileserver"; - - nfs = { - enable = mkEnableOption "Enable nfs fileserver"; - exports = mkOption { - type = types.str; - default = ""; - }; - }; - - glusterfs.enable = mkEnableOption "Enable glusterfs fileserver"; - }; - - config = mkIf cfg.enable ( - mkMerge [ - common - (mkIf cfg.nfs.enable nfs) - (mkIf cfg.glusterfs.enable glusterfs) - ] - ); - - imports = [ ./os.nix ]; -} diff --git a/modules.bak/initca.nix b/modules.bak/initca.nix deleted file mode 100644 index 325a954..0000000 --- a/modules.bak/initca.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - pkgs ? import {}, - ca ? null, - name ? "ca", - algo ? "rsa", - hosts ? [], - ...}: -with pkgs; -let - ca_csr = pkgs.writeText "${name}-csr.json" (builtins.toJSON { - inherit hosts; - CN = "${name}"; - key = { - inherit algo; - size = if algo == "ecdsa" then 256 else 2048; - }; - names = [ - { - CN = "${name}"; - O = "NixOS"; - OU = "${name}.pki.caSpec"; - L = "generated"; - } - ]; - } - ); - ca' = - pkgs.runCommand "initca" { - buildInputs = [ pkgs.cfssl ]; - } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; - mkdir -p $out; cp *.pem $out ''; - initca = if ca != null then ca else ca'; -in - # make ca derivation sha depend on initca cfssl output - pkgs.stdenv.mkDerivation { - inherit name; - src = initca; - buildCommand = '' - mkdir -p $out; - cp -r $src/* $out - ''; - } diff --git a/modules.bak/k8s.nix b/modules.bak/k8s.nix deleted file mode 100644 index d32cf34..0000000 --- a/modules.bak/k8s.nix +++ /dev/null @@ -1,384 +0,0 @@ -{ pkgs, lib, config, ...}: -with lib; -let - cfg = config.cluster; - - pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; - - masterAddress = cfg.k8s.master.address; - - apiserverAddress = "https://${masterAddress}:4443"; - - cfssl-apitoken = - let - apitoken = pkgs.stdenv.mkDerivation { - name = "apitoken"; - buildCommand = '' - head -c ${toString (32 / 2)} /dev/urandom | \ - od -An -t x | tr -d ' ' > $out - chmod 400 $out - ''; - }; - in - # make ca derivation sha depend on initca cfssl output - pkgs.stdenv.mkDerivation { - name = "cfssl-apitoken"; - src = apitoken; - buildCommand = '' - cp $src $out - ''; - }; - - cluster-scripts = - let - first = builtins.head cfg.k8s.ingressNodes; - rest = builtins.tail cfg.k8s.ingressNodes; - ingressNodes = builtins.foldl' (a: x: - a + ",${x}") first rest; - ingressReplicaCount = - builtins.toString (builtins.length cfg.k8s.ingressNodes); - show-kubernetes-charts-config = '' - #!/usr/bin/env bash - cat << EOF - # Generated by show-kubernetes-charts-config - # $(date) - # Charts in ${kubernetes-charts}/share/kubernetes-charts - - vars=( - initca="${pki.initca}" - apiserver="${cfg.k8s.master.name}" - cluster="${cfg.clusterName}" - ingress_nodes="${ingressNodes}" - ingress_replica_count="${ingressReplicaCount}" - fileserver="${cfg.k8s.fileserver}" - acme_email="${cfg.k8s.charts.acme_email}" - grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)" - grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)" - ) - - # -------------------- no modifications below -------------------- - EOF - cat << '"'"'EOF'"'"' - apply=''${KUBECTL_CMD:-apply} - - substitute_all () { - local x i k v subs - x="$(/dev/null 2>&1 || kubectl create ns $namespace - } - - export_vars () { - local i - for i in "''${vars[@]}"; do eval "$i"; done - } - - kubectl_apply () { - local x - x="$( \$dest/config.sh - ''; - in - pkgs.stdenv.mkDerivation { - name = "cluster-scripts"; - src = ../scripts; - buildCommand = '' - mkdir -p $out/bin - cp $src/* $out/bin - echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config - chmod a+x $out/bin/show-kubernetes-charts-config - echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts - chmod a+x $out/bin/copy-kubernetes-charts - ''; - }; - - kubernetes-charts = pkgs.stdenv.mkDerivation rec { - name = "kubernetes-charts"; - src = ../charts; - buildCommand = '' - mkdir -p $out/share/${name} - cp -r $src/* $out/share/${name} - ''; - }; - - install-apitoken = '' - #!${pkgs.bash}/bin/bash - set -e - if [ -d /var/lib/cfssl ]; then - cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret - chown cfssl /var/lib/cfssl/apitoken.secret - chmod 640 /var/lib/cfssl/apitoken.secret - else - mkdir -p /var/lib/kubernetes/secrets - cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret - chown root /var/lib/kubernetes/secrets/apitoken.secret - chmod 600 /var/lib/kubernetes/secrets/apitoken.secret - fi - ''; - - kubeMaster = { - services.cfssl.ca = pki.ca.cert; - services.cfssl.caKey = pki.ca.key; - services.kubernetes = { - roles = [ "master" ]; - inherit apiserverAddress; - masterAddress = "${cfg.k8s.master.name}.${cfg.domain}"; - clusterCidr = cfg.k8s.cidr; - pki.genCfsslCACert = false; - pki.genCfsslAPIToken = false; - pki.caCertPathPrefix = "${pki.initca}/ca"; - - kubelet = { - # clusterDomain = "${cfg.clusterName}.local"; - }; - - apiserver = { - advertiseAddress = masterAddress; - authorizationMode = [ "Node" "RBAC" ]; - allowPrivileged = true; - securePort = 4443; - insecurePort = 8080; - extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}"; - extraSANs = cfg.k8s.master.extraSANs; - # verbosity = 4; - }; - - controllerManager = { - bindAddress = masterAddress; - extraOpts = "--authorization-always-allow-paths=/healthz,/metrics"; - }; - - scheduler.address = masterAddress; - - addonManager.enable = true; - addons = { - dns = { - enable = true; - # clusterDomain = "${cfg.clusterName}.local"; - reconcileMode = "EnsureExists"; - }; - }; - }; - - services.etcd = { - listenClientUrls = [ "https://${masterAddress}:2379" ]; - }; - - networking.firewall = { - allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ]; - allowedUDPPorts = [ 53 4053 ]; - }; - - environment.systemPackages = [ - pkgs.kubernetes-helm - pkgs.kubectl - cluster-scripts - kubernetes-charts - ]; - - systemd.services.kube-certmgr-apitoken-bootstrap = { - description = "Kubernetes certmgr bootstrapper"; - wantedBy = [ "cfssl.service" ]; - before = [ "cfssl.target" ]; - script = install-apitoken; - serviceConfig = { - RestartSec = "10s"; - Restart = "on-failure"; - }; - }; - - # systemd.services.cfssl-restart = { - # enable = true; - # startAt = "00/6:00"; - # description = "Restrart cfssl which regularly locks up"; - # script = "systemctl restart cfssl.service"; - # }; - - systemd.services.kube-socat-https-proxy = { - enable = true; - wantedBy = [ "kubernetes.target" ]; - after = [ "kubelet.target" ]; - description = "Proxy TCP port 443 to ingress NodePort at 30443"; - script = "${pkgs.socat}/bin/socat TCP-LISTEN:443,fork,reuseaddr TCP:127.0.0.1:30443"; - serviceConfig = { - RestartSec = "10s"; - Restart = "on-failure"; - }; - }; - }; - - kubeNode = { - services.kubernetes = rec { - roles = [ "node" ]; - inherit apiserverAddress; - # masterAddress = cfg.k8s.master.name; - masterAddress = "${cfg.k8s.master.name}.${cfg.domain}"; - clusterCidr = cfg.k8s.cidr; - # kubelet.clusterDomain = "${cfg.clusterName}.local"; - kubelet.hostname = "${cfg.hostName}.${cfg.domain}"; - proxy.hostname = "${cfg.hostName}.${cfg.domain}"; - }; - - networking = { - firewall = { - enable = true; - allowedTCPPorts = [ 4194 10250 ]; - allowedUDPPorts = [ 53 ]; - }; - }; - virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; - virtualisation.docker.autoPrune.enable = true; - systemd.services.kube-certmgr-apitoken-bootstrap = { - description = "Kubernetes certmgr bootstrapper"; - wantedBy = [ "certmgr.service" ]; - before = [ "certmgr.service" ]; - script = install-apitoken; - serviceConfig = { - RestartSec = "10s"; - Restart = "on-failure"; - }; - }; - }; - -in { - options.cluster.k8s = { - enable = mkEnableOption "Enable kubernetes"; - nodes = mkOption { - type = types.listOf types.attrs; - default = []; - }; - - fileserver = mkOption { - type = types.str; - default = null; - }; - - cidr = mkOption { - type = types.str; - default = "10.0.0.0/16"; - }; - - ingressNodes = mkOption { - type = types.listOf types.str; - default = null; - }; - - master = { - enable = mkEnableOption "Enable kubernetes master node"; - - address = mkOption { - type = types.str; - default = null; - }; - - name = mkOption { - type = types.str; - default = null; - }; - - extraSANs = mkOption { - type = types.listOf types.str; - default = []; - }; - - hw = mkOption { - type = types.path; - default = null; - }; - }; - - node = { - enable = mkEnableOption "Enable kubernetes"; - }; - - charts = { - acme_email = mkOption { - type = types.str; - default = ""; - }; - - grafana_smtp_user = mkOption { - type = types.str; - default = ""; - }; - - grafana_smtp_password = mkOption { - type = types.str; - default = ""; - }; - }; - }; - - config = mkIf cfg.k8s.enable ( - mkMerge [ - (mkIf cfg.k8s.master.enable kubeMaster) - (mkIf cfg.k8s.node.enable kubeNode) - ] - ); - - imports = [ ./os.nix ]; - -} - diff --git a/modules.bak/modules.nix b/modules.bak/modules.nix deleted file mode 100644 index 2090f14..0000000 --- a/modules.bak/modules.nix +++ /dev/null @@ -1,7 +0,0 @@ -{ - imports = [ - ./os.nix - ./fs.nix - ./k8s.nix - ]; -} diff --git a/modules.bak/os.nix b/modules.bak/os.nix deleted file mode 100644 index ac57fbd..0000000 --- a/modules.bak/os.nix +++ /dev/null @@ -1,118 +0,0 @@ -{ pkgs, lib, config, ... }: -with lib; -let - cfg = config.cluster; - - pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; }; - - common = { - users.extraUsers.admin.openssh.authorizedKeys.keys = - cfg.adminAuthorizedKeys; - - users.extraUsers.root.openssh.authorizedKeys.keys = - cfg.adminAuthorizedKeys; - - networking = { - domain = cfg.domain; - search = cfg.searchDomains; - extraHosts = cfg.extraHosts; - firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; - firewall.allowedTCPPorts = [ 80 443 111 ]; - firewall.allowedUDPPorts = [ 111 24007 24008 ]; - } // ( - if cfg.externalInterface == null then - { hostName = cfg.hostName; } - else { - hostName = cfg.hostName; - interfaces."${cfg.externalInterface}" = { - useDHCP = false; - ipv4.addresses = [ { - address = cfg.address; - prefixLength = 24; - } ]; - }; - defaultGateway = cfg.defaultGateway; - nameservers = cfg.nameservers; - } - ); - - security.pki.certificateFiles = [ pki.ca.cert ]; - boot.kernel.sysctl = { - "kernel.mm.transparent_hugepage.enabled" = "never"; - "net.core.somaxconn" = "512"; - }; - - environment.systemPackages = with pkgs; [ - nfs-utils - ]; - }; -in -{ - options.cluster = { - initca = mkOption { - type = types.path; - }; - - hostName = mkOption { - type = types.nullOr types.str; - default = null; - }; - - address = mkOption { - type = types.nullOr types.str; - default = null; - }; - - externalInterface = mkOption { - type = types.nullOr types.str; - default = null; - }; - - - defaultGateway = mkOption { - type = types.nullOr types.str; - default = null; - }; - - nameservers = mkOption { - type = types.listOf types.str; - default = [ "8.8.8.8" ]; - }; - - domain = mkOption { - type = types.str; - default = null; - }; - - searchDomains = mkOption { - type = types.listOf types.str; - default = [ cfg.domain ]; - }; - - cert = mkOption { - type = types.attrs; - default = null; - }; - - clusterName = mkOption { - type = types.str; - default = null; - }; - - extraHosts = mkOption { - type = types.str; - }; - - adminAuthorizedKeys = mkOption { - type = types.listOf types.str; - default = []; - }; - }; - - config = common; - - imports = [ - ../nixos/configuration.nix - ]; - -} diff --git a/modules.bak/pki.nix b/modules.bak/pki.nix deleted file mode 100644 index b73706c..0000000 --- a/modules.bak/pki.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ pkgs, ca ? "", algo ? "rsa" }: -let - initca = import ./initca.nix { inherit pkgs ca; }; - - ca' = { - key = "${initca}/ca-key.pem"; - cert = "${initca}/ca.pem"; - }; - - ca-config = pkgs.writeText "ca-config.json" '' - { - "signing": { - "default": { - "expiry": "8760h" - }, - "profiles": { - "default": { - "usages": [ - "signing", - "key encipherment", - "server auth", - "client auth" - ], - "expiry": "8760h" - } - } - } - } - ''; - - gencsr = args: - let - csr = { - CN = "${args.cn}"; - key = { - inherit algo; - size = if algo == "ecdsa" then 256 else 2048; - }; - names = [ - { - CN = "${args.cn}"; - O = "${args.o}"; - OU = "${args.cn}.${args.o}.pki.caSpec"; - L = "generated"; - } - ]; - hosts = args.hosts; - }; - in - pkgs.writeText "${args.cn}-csr.json" (builtins.toJSON csr); -in -# Example usage: -# -# gencert { cn = "test"; ca = ca; o = "test; }; -# -rec { - inherit initca; - ca = ca'; - gencert = attrs: - let - conf = { - cn = attrs.cn; - ca = attrs.ca; - csr = gencsr { cn = attrs.cn; o = attrs.o; hosts = attrs.hosts; }; - }; - cfssl = conf: - '' - cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \ - -config=${ca-config} -profile=default ${conf.csr} | \ - cfssljson -bare cert; \ - mkdir -p $out; cp *.pem $out - ''; - crt = - pkgs.runCommand "${attrs.cn}" { - buildInputs = [ pkgs.cfssl ]; - } (cfssl conf); - in - { - key = "${crt}/cert-key.pem"; - cert = "${crt}/cert.pem"; - }; -} diff --git a/overrides/kubelet.nix b/overrides/kubelet.nix deleted file mode 100644 index 2d03cb9..0000000 --- a/overrides/kubelet.nix +++ /dev/null @@ -1,359 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; - -let - top = config.services.kubernetes; - cfg = top.kubelet; - - cniConfig = - if cfg.cni.config != [] && cfg.cni.configDir != null then - throw "Verbatim CNI-config and CNI configDir cannot both be set." - else if cfg.cni.configDir != null then - cfg.cni.configDir - else - (pkgs.buildEnv { - name = "kubernetes-cni-config"; - paths = imap (i: entry: - pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) - ) cfg.cni.config; - }); - - infraContainer = pkgs.dockerTools.buildImage { - name = "pause"; - tag = "latest"; - contents = top.package.pause; - config.Cmd = "/bin/pause"; - }; - - kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig; - - manifestPath = "kubernetes/manifests"; - - taintOptions = with lib.types; { name, ... }: { - options = { - key = mkOption { - description = "Key of taint."; - default = name; - type = str; - }; - value = mkOption { - description = "Value of taint."; - type = str; - }; - effect = mkOption { - description = "Effect of taint."; - example = "NoSchedule"; - type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"]; - }; - }; - }; - - taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints); -in -{ - disabledModules = [ "services/cluster/kubernetes/kubelet.nix" ]; - - imports = [ - (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "") - (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "") - (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "") - ]; - - ###### interface - options.services.kubernetes.kubelet = with lib.types; { - - address = mkOption { - description = "Kubernetes kubelet info server listening address."; - default = "0.0.0.0"; - type = str; - }; - - clusterDns = mkOption { - description = "Use alternative DNS."; - default = "10.1.0.1"; - type = str; - }; - - clusterDomain = mkOption { - description = "Use alternative domain."; - default = config.services.kubernetes.addons.dns.clusterDomain; - type = str; - }; - - extraSANs = mkOption { - description = "Extra x509 Subject Alternative Names to be added to the kubelet tls cert."; - default = []; - type = listOf str; - }; - - clientCaFile = mkOption { - description = "Kubernetes apiserver CA file for client authentication."; - default = top.caFile; - type = nullOr path; - }; - - cni = { - packages = mkOption { - description = "List of network plugin packages to install."; - type = listOf package; - default = []; - }; - - config = mkOption { - description = "Kubernetes CNI configuration."; - type = listOf attrs; - default = []; - example = literalExample '' - [{ - "cniVersion": "0.3.1", - "name": "mynet", - "type": "bridge", - "bridge": "cni0", - "isGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "subnet": "10.22.0.0/16", - "routes": [ - { "dst": "0.0.0.0/0" } - ] - } - } { - "cniVersion": "0.3.1", - "type": "loopback" - }] - ''; - }; - - configDir = mkOption { - description = "Path to Kubernetes CNI configuration directory."; - type = nullOr path; - default = null; - }; - }; - - enable = mkEnableOption "Kubernetes kubelet."; - - extraOpts = mkOption { - description = "Kubernetes kubelet extra command line options."; - default = ""; - type = str; - }; - - featureGates = mkOption { - description = "List set of feature gates"; - default = top.featureGates; - type = listOf str; - }; - - healthz = { - bind = mkOption { - description = "Kubernetes kubelet healthz listening address."; - default = "127.0.0.1"; - type = str; - }; - - port = mkOption { - description = "Kubernetes kubelet healthz port."; - default = 10248; - type = int; - }; - }; - - hostname = mkOption { - description = "Kubernetes kubelet hostname override."; - default = config.networking.hostName; - type = str; - }; - - kubeconfig = top.lib.mkKubeConfigOptions "Kubelet"; - - manifests = mkOption { - description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; - type = attrsOf attrs; - default = {}; - }; - - networkPlugin = mkOption { - description = "Network plugin to use by Kubernetes."; - type = nullOr (enum ["cni" "kubenet"]); - default = "kubenet"; - }; - - nodeIp = mkOption { - description = "IP address of the node. If set, kubelet will use this IP address for the node."; - default = null; - type = nullOr str; - }; - - registerNode = mkOption { - description = "Whether to auto register kubelet with API server."; - default = true; - type = bool; - }; - - port = mkOption { - description = "Kubernetes kubelet info server listening port."; - default = 10250; - type = int; - }; - - seedDockerImages = mkOption { - description = "List of docker images to preload on system"; - default = []; - type = listOf package; - }; - - taints = mkOption { - description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)."; - default = {}; - type = attrsOf (submodule [ taintOptions ]); - }; - - tlsCertFile = mkOption { - description = "File containing x509 Certificate for HTTPS."; - default = null; - type = nullOr path; - }; - - tlsKeyFile = mkOption { - description = "File containing x509 private key matching tlsCertFile."; - default = null; - type = nullOr path; - }; - - unschedulable = mkOption { - description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; - default = false; - type = bool; - }; - - verbosity = mkOption { - description = '' - Optional glog verbosity level for logging statements. See - - ''; - default = null; - type = nullOr int; - }; - - }; - - ###### implementation - config = mkMerge [ - (mkIf cfg.enable { - services.kubernetes.kubelet.seedDockerImages = [infraContainer]; - - systemd.services.kubelet = { - description = "Kubernetes Kubelet Service"; - wantedBy = [ "kubernetes.target" ]; - after = [ "network.target" "docker.service" "kube-apiserver.service" ]; - path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path; - preStart = '' - ${concatMapStrings (img: '' - echo "Seeding docker image: ${img}" - docker load <${img} - '') cfg.seedDockerImages} - - rm /opt/cni/bin/* || true - ${concatMapStrings (package: '' - echo "Linking cni package: ${package}" - ln -fs ${package}/bin/* /opt/cni/bin - '') cfg.cni.packages} - ''; - serviceConfig = { - Slice = "kubernetes.slice"; - CPUAccounting = true; - MemoryAccounting = true; - Restart = "on-failure"; - RestartSec = "1000ms"; - ExecStart = ''${top.package}/bin/kubelet \ - --address=${cfg.address} \ - --authentication-token-webhook \ - --authentication-token-webhook-cache-ttl="10s" \ - --authorization-mode=Webhook \ - ${optionalString (cfg.clientCaFile != null) - "--client-ca-file=${cfg.clientCaFile}"} \ - ${optionalString (cfg.clusterDns != "") - "--cluster-dns=${cfg.clusterDns}"} \ - ${optionalString (cfg.clusterDomain != "") - "--cluster-domain=${cfg.clusterDomain}"} \ - --cni-conf-dir=${cniConfig} \ - ${optionalString (cfg.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ - --hairpin-mode=hairpin-veth \ - --healthz-bind-address=${cfg.healthz.bind} \ - --healthz-port=${toString cfg.healthz.port} \ - --hostname-override=${cfg.hostname} \ - --kubeconfig=${kubeconfig} \ - ${optionalString (cfg.networkPlugin != null) - "--network-plugin=${cfg.networkPlugin}"} \ - ${optionalString (cfg.nodeIp != null) - "--node-ip=${cfg.nodeIp}"} \ - --pod-infra-container-image=pause \ - ${optionalString (cfg.manifests != {}) - "--pod-manifest-path=/etc/${manifestPath}"} \ - --port=${toString cfg.port} \ - --register-node=${boolToString cfg.registerNode} \ - ${optionalString (taints != "") - "--register-with-taints=${taints}"} \ - --root-dir=${top.dataDir} \ - ${optionalString (cfg.tlsCertFile != null) - "--tls-cert-file=${cfg.tlsCertFile}"} \ - ${optionalString (cfg.tlsKeyFile != null) - "--tls-private-key-file=${cfg.tlsKeyFile}"} \ - ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ - ${cfg.extraOpts} - ''; - WorkingDirectory = top.dataDir; - }; - }; - - # Allways include cni plugins - services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins]; - - boot.kernelModules = ["br_netfilter"]; - - services.kubernetes.kubelet.hostname = with config.networking; - mkDefault (hostName + optionalString (domain != null) ".${domain}"); - - services.kubernetes.pki.certs = with top.lib; { - kubelet = mkCert { - name = "kubelet"; - CN = top.kubelet.hostname; - hosts = top.kubelet.extraSANs; - action = "systemctl restart kubelet.service"; - - }; - kubeletClient = mkCert { - name = "kubelet-client"; - CN = "system:node:${top.kubelet.hostname}"; - fields = { - O = "system:nodes"; - }; - action = "systemctl restart kubelet.service"; - }; - }; - - services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress; - }) - - (mkIf (cfg.enable && cfg.manifests != {}) { - environment.etc = mapAttrs' (name: manifest: - nameValuePair "${manifestPath}/${name}.json" { - text = builtins.toJSON manifest; - mode = "0755"; - } - ) cfg.manifests; - }) - - (mkIf (cfg.unschedulable && cfg.enable) { - services.kubernetes.kubelet.taints.unschedulable = { - value = "true"; - effect = "NoSchedule"; - }; - }) - - ]; -}