{ pkgs, lib, settings, here, ...}: with lib; let cluster-ca = import ./initca.nix { inherit pgks; }; cfssl-apitoken = pkgs.stdenv.mkDerivation { name = "cfssl-apitoken"; buildCommand = '' head -c ${toString (32 / 2)} /dev/urandom | \ od -An -t x | tr -d ' ' > $out chmod 400 $out ''; }; kube-system-bootstrap = pkgs.stdenv.mkDerivation { name = "kube-system-bootstrap"; src = ./kube-system-bootstrap; buildCommand = '' mkdir -p $out cp -r $src/* $out ''; }; bootstrap-kube-system-sh = pkgs.writeScriptBin "bootstrap-kube-system.sh" '' #!${pkgs.bash}/bin/bash cd ${kube-system-bootstrap} ${pkgs.bash}/bin/bash ./kube-system-bootstrap ${cluster-ca} ${settings.clusterName} ''; kube-scripts = pkgs.stdenv.mkDerivation { name = "kube-scripts"; buildCommand = '' mkdir -p $out/bin cd $out/bin ln -s ${kube-system-bootstrap}/bin/* . ''; }; install-apitoken = '' #!${pkgs.bash}/bin/bash set -e if [ -d /var/lib/cfssl ]; then cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret chown cfssl /var/lib/cfssl/apitoken.secret chmod 640 /var/lib/cfssl/apitoken.secret else mkdir -p /var/lib/kubernetes/secrets cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret chown root /var/lib/kubernetes/secrets/apitoken.secret chmod 600 /var/lib/kubernetes/secrets/apitoken.secret fi ''; cidr = "10.10.0.0/16"; in rec { kubeMaster = { services.cfssl.ca = "${cluster-ca}/ca.pem"; services.cfssl.caKey = "${cluster-ca}/ca-key.pem"; services.kubernetes = { roles = [ "master" ]; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; clusterCidr = cidr; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${cluster-ca}/ca"; kubelet = { unschedulable = false; clusterDomain = "${settings.clusterName}.local"; }; apiserver = { advertiseAddress = settings.masterAddress; authorizationMode = [ "Node" "RBAC" ]; securePort = 4443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem"; }; addons = { dns = { enable = true; clusterDomain = "${settings.clusterName}.local"; reconcileMode = "EnsureExists"; }; }; }; networking.firewall = { allowedTCPPorts = [ 53 5000 8080 4443 ]; #;4053 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm pkgs.kubectl kube-scripts bootstrap-kube-system-sh ]; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "cfssl.service" ]; before = [ "cfssl.target" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; Restart = "on-failure"; }; }; }; kubeWorker = { services.kubernetes = rec { roles = [ "node" ]; clusterCidr = cidr; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; kubelet.clusterDomain = "${settings.clusterName}.local"; }; networking = { firewall = { enable = true; allowedTCPPorts = [ 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "certmgr.service" ]; before = [ "certmgr.service" ]; script = install-apitoken; serviceConfig = { RestartSec = "10s"; Restart = "on-failure"; }; }; }; baseNixos = name: { users.extraUsers.admin.openssh.authorizedKeys.keys = settings.adminAuthorizedKeys; imports = [ ./nixos/configuration.nix (here + "/${name}.nix") ]; security.pki.certificateFiles = [ "${cluster-ca}/ca.pem" ]; # services.glusterfs = { # enable = true; # # tlsSettings = { # # caCert = certs.ca.caFile; # # tlsKeyPath = certs.self.keyFile; # # tlsPem = certs.self.certFile; # }; # }; networking = { hostName = name; extraHosts = settings.clusterHosts; # nameservers = [ masterAddress ]; # dhcpcd.extraConfig = '' # static domain_name_servers=${masterAddress} # ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; environment.systemPackages = with pkgs; [ nfs-utils ]; }; apiserver = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) kubeMaster ]; }; worker = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) kubeWorker ]; }; host = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) ]; }; }