here: { pkgs, lib, settings, ...}: with lib; let cluster-ca = pkgs.stdenv.mkDerivation { name = "cluster-ca"; src = here + /ca; buildCommand = '' mkdir -p $out cp $src/* $out ''; }; cfssl-apitoken = pkgs.stdenv.mkDerivation { name = "cfssl-apitoken"; buildCommand = '' head -c ${toString (32 / 2)} /dev/urandom | \ od -An -t x | tr -d ' ' > $out chmod 400 $out ''; }; nixos-kubernetes-join-nodes = workers: let wrk = builtins.foldl' (a: s: a + " " + s) "" workers; in pkgs.writeScriptBin "nixos-kubernetes-join-nodes" '' #!/bin/sh set -e token=$(cat /var/lib/cfssl/apitoken.secret) for i in ${wrk}; do ssh root@$i "echo $token | sh nixos-kubernetes-node-join" done ''; kube-system-bootstrap = pkgs.stdenv.mkDerivation { name = "kube-system-bootstrap"; src = ../kube-system-bootstrap; buildCommand = '' mkdir -p $out/bin mkdir -p $out/share/kube-system-bootstrap cp -r $src/* $out/share/kube-system-bootstrap/ cd $out/bin ln -s ../share/kube-system-bootstrap/bin/* . ln -s ../share/kube-system-bootstrap/kube-system-bootstrap . ''; }; install-apitoken = pkgs.writeScript "kube-install-certmgr-apitoken" '' #!${pkgs.bash}/bin/bash set -e if [ -d /var/lib/cfssl ]; then cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret chown cfssl /var/lib/cfssl/apitoken.secret chmod 600 /var/lib/cfssl/apitoken.secret fi cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret chown root /var/lib/kubernetes/secrets/apitoken.secret chmod 600 /var/lib/kubernetes/secrets/apitoken.secret ''; cidr = "10.10.0.0/16"; in rec { kubeMaster = { services.cfssl.ca = "${cluster-ca}/ca.pem"; services.cfssl.caKey = "${cluster-ca}/ca-key.pem"; services.kubernetes = { roles = [ "master" ]; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; clusterCidr = cidr; kubelet.unschedulable = false; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${cluster-ca}/ca"; apiserver = { advertiseAddress = settings.masterAddress; authorizationMode = [ "Node" "RBAC" ]; securePort = 8443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem"; }; addons = { dns = { enable = true; # clusterDomain = "local"; reconcileMode = "EnsureExists"; }; }; }; networking.firewall = { allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm (nixos-kubernetes-join-nodes settings.workers) kube-system-bootstrap ]; }; kubeWorker = { services.kubernetes = rec { roles = [ "node" ]; clusterCidr = cidr; masterAddress = settings.master; apiserverAddress = settings.apiserverAddress; }; networking = { firewall = { enable = true; allowedTCPPorts = [ 4194 10250 ]; allowedUDPPorts = [ 53 ]; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; }; }; virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; virtualisation.docker.autoPrune.enable = true; }; baseNixos = name: { imports = [ ../nixos/configuration.nix (here + "/${name}.nix") ]; security.pki.certificateFiles = [ "${cluster-ca}/ca.pem" ]; # services.glusterfs = { # enable = true; # # tlsSettings = { # # caCert = certs.ca.caFile; # # tlsKeyPath = certs.self.keyFile; # # tlsPem = certs.self.certFile; # }; # }; networking = { hostName = name; extraHosts = settings.clusterHosts; # nameservers = [ masterAddress ]; # dhcpcd.extraConfig = '' # static domain_name_servers=${masterAddress} # ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; }; users.extraUsers.admin.openssh.authorizedKeys.keys = settings.adminAuthorizedKeys; systemd.services.kube-certmgr-apitoken-bootstrap = { description = "Kubernetes certmgr bootstrapper"; wantedBy = [ "cfssl.service" ]; before = [ "cfssl.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${install-apitoken}"; }; }; }; apiserver = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) kubeMaster ]; }; worker = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) kubeWorker ]; }; host = ip: name: self: { deployment.targetHost = ip; require = [ (baseNixos name) ]; }; }