{ pkgs, lib, settings, ...}: with lib; let cluster-ca = pkgs.stdenv.mkDerivation { name = "cluster-ca"; src = ./ca; buildCommand = '' mkdir -p $out cp $src/* $out ''; }; nixos-kubernetes-join-nodes = workers: let wrk = builtins.foldl' (a: s: a + " " + s) "" workers; in pkgs.writeScriptBin "nixos-kubernetes-join-nodes" '' #!/bin/sh set -e token=$(cat /var/lib/cfssl/apitoken.secret) for i in ${wrk}; do ssh root@$i "echo $token | sh nixos-kubernetes-node-join" done ''; cidr = "10.10.0.0/16"; in rec { kubeMaster = { services.cfssl.ca = "${cluster-ca}/ca.pem"; services.cfssl.caKey = "${cluster-ca}/ca-key.pem"; services.kubernetes = { roles = [ "master" ]; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; clusterCidr = cidr; kubelet.unschedulable = false; pki.genCfsslCACert = false; pki.caCertPathPrefix = "${cluster-ca}/ca"; apiserver = { advertiseAddress = settings.masterAddress; authorizationMode = [ "Node" "RBAC" ]; securePort = 8443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem"; }; addons = { dns = { enable = true; # clusterDomain = "local"; reconcileMode = "EnsureExists"; }; }; }; networking.firewall = { allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm (nixos-kubernetes-join-nodes settings.workers) ]; }; kubeWorker = { services.kubernetes = rec { roles = [ "node" ]; clusterCidr = cidr; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; }; networking = { firewall = { enable = true; allowedTCPPorts = [ 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; }; baseNixos = name: { imports = [ (../nixos/hardware-configuration + "/${name}.nix") ../nixos/configuration.nix ]; security.pki.certificateFiles = [ "${cluster-ca}/ca.pem" ]; # services.glusterfs = { # enable = true; # # tlsSettings = { # # caCert = certs.ca.caFile; # # tlsKeyPath = certs.self.keyFile; # # tlsPem = certs.self.certFile; # }; # }; networking = { hostName = name; extraHosts = settings.clusterHosts; # nameservers = [ masterAddress ]; # dhcpcd.extraConfig = '' # static domain_name_servers=${masterAddress} # ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; }; apiserver = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) kubeMaster ]; }; worker = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) kubeWorker ]; }; host = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) ]; }; }