From a3fa5ebc3683d655e9f341c650ddf9becfea298a Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Tue, 17 Dec 2019 22:12:50 +0100 Subject: [PATCH] Multiple improvements and refactorizations --- lib/initca.nix | 5 +- lib/k8s.nix | 139 +++++++++++++++++++++++++++++++------------------ lib/nixos | 2 +- 3 files changed, 91 insertions(+), 55 deletions(-) diff --git a/lib/initca.nix b/lib/initca.nix index 04cb94c..a679e39 100644 --- a/lib/initca.nix +++ b/lib/initca.nix @@ -1,4 +1,4 @@ -{ pkgs ? import {}, ...}: +{ pkgs ? import {}, initca ? "", ...}: with pkgs; let initca' = @@ -22,11 +22,12 @@ let buildInputs = [ pkgs.cfssl ]; } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \ mkdir -p $out; cp *.pem $out''; + ca = if initca != "" then initca else initca'; in # make ca derivation sha depend on initca cfssl output pkgs.stdenv.mkDerivation { name = "ca"; - src = initca'; + src = ca; buildCommand = '' mkdir -p $out; cp -r $src/* $out diff --git a/lib/k8s.nix b/lib/k8s.nix index 3781f1a..e6ed34e 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -1,7 +1,11 @@ -{ pkgs, lib, settings, here, ...}: +{ pkgs, lib, settings, here ? "", ...}: with lib; let - cluster-ca = import ./initca.nix { inherit pgks; }; + apiserverAddress = "https://${masterAddress}:4443"; + masterAddress = settings.master.address; + initca = settings.initca; + + cluster-ca = import ./initca.nix { inherit pgks initca; }; cfssl-apitoken = pkgs.stdenv.mkDerivation { name = "cfssl-apitoken"; @@ -12,30 +16,50 @@ let ''; }; - kube-system-bootstrap = pkgs.stdenv.mkDerivation { - name = "kube-system-bootstrap"; - src = ./kube-system-bootstrap; - buildCommand = '' - mkdir -p $out - cp -r $src/* $out - cp ${here}/bootstrap.conf $out/${settings.clusterName}.conf - ''; - }; + kube-system-bootstrap = + with settings; + let + worker_nodes = pkgs.writeText "worker-nodes.txt" ( + builtins.foldl' (a: x: + a + " - ${x.address}\n" + ) "" settings.workers); + grafana_ldap = pkgs.writeText "grafana-ldap.toml" grafana_ldap_toml; + in + pkgs.stdenv.mkDerivation { + name = "bootstrap-kube-system"; + src = ../bootstrap; + buildCommand = '' + mkdir -p $out/bin + mkdir -p $out/share/kube-system-bootstrap/config + mkdir -p $out/share/kube-system-bootstrap/charts - bootstrap-kube-system-sh = pkgs.writeScriptBin "bootstrap-kube-system.sh" '' - #!${pkgs.bash}/bin/bash - cd ${kube-system-bootstrap} - ${pkgs.bash}/bin/bash ./kube-system-bootstrap ${cluster-ca} ${settings.clusterName} - ''; + export bash="${pkgs.bash}" + export apiserver="${settings.master.address}" + export initca="${initca}" + export cluster="${clusterName}" + export fileserver="${fileserver}" + export acme_email="${acme_email}" + export grafana_smtp_user="$(echo -n ${grafana_smtp_user} | base64 -w0)" + export grafana_smtp_password="$(echo -n ${grafana_smtp_password} | base64 -w0)" + export grafana_ldap_toml="$(cat ${grafana_ldap} | base64 -w0)" + export workers="$(cat ${worker_nodes})" - kube-scripts = pkgs.stdenv.mkDerivation { - name = "kube-scripts"; - buildCommand = '' - mkdir -p $out/bin - cd $out/bin - ln -s ${kube-system-bootstrap}/bin/* . - ''; - }; + substituteAll $src/kube-system-bootstrap $out/bin/bootstrap-kube-system + chmod 755 $out/bin/bootstrap-kube-system + + cd $src/config + for i in *; do + substituteAll $i $out/share/kube-system-bootstrap/config/$i + done + + cd $src/charts + for i in *; do + substituteAll $i $out/share/kube-system-bootstrap/charts/$i + done + + cp $src/bin/* $out/bin + ''; + }; install-apitoken = '' #!${pkgs.bash}/bin/bash @@ -52,34 +76,38 @@ let fi ''; - cidr = "10.10.0.0/16"; -in -rec { kubeMaster = { services.cfssl.ca = "${cluster-ca}/ca.pem"; services.cfssl.caKey = "${cluster-ca}/ca-key.pem"; services.kubernetes = { roles = [ "master" ]; - masterAddress = settings.master; - apiserverAddress = settings.apiserverAddress; - clusterCidr = cidr; + inherit apiserverAddress masterAddress; + clusterCidr = settings.cidr; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; pki.caCertPathPrefix = "${cluster-ca}/ca"; kubelet = { - unschedulable = false; clusterDomain = "${settings.clusterName}.local"; }; apiserver = { - advertiseAddress = settings.masterAddress; + advertiseAddress = masterAddress; authorizationMode = [ "Node" "RBAC" ]; + allowPrivileged = true; securePort = 4443; insecurePort = 8080; extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem"; }; + controllerManager = { + bindAddress = masterAddress; + extraOpts = "--authorization-always-allow-paths=/healthz,/metrics"; + }; + + scheduler.address = masterAddress; + + addonManager.enable = true; addons = { dns = { enable = true; @@ -89,16 +117,19 @@ rec { }; }; + services.etcd = { + listenClientUrls = [ "https://${masterAddress}:2379" ]; + }; + networking.firewall = { - allowedTCPPorts = [ 53 5000 8080 4443 ]; #;4053 ]; + allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ]; allowedUDPPorts = [ 53 4053 ]; }; environment.systemPackages = [ pkgs.kubernetes-helm pkgs.kubectl - kube-scripts - bootstrap-kube-system-sh + kube-system-bootstrap ]; systemd.services.kube-certmgr-apitoken-bootstrap = { @@ -116,9 +147,8 @@ rec { kubeWorker = { services.kubernetes = rec { roles = [ "node" ]; - clusterCidr = cidr; - masterAddress = settings.master; - apiserverAddress = settings.apiserverAddress; + inherit apiserverAddress masterAddress; + clusterCidr = settings.cidr; kubelet.clusterDomain = "${settings.clusterName}.local"; }; @@ -148,6 +178,11 @@ rec { users.extraUsers.admin.openssh.authorizedKeys.keys = settings.adminAuthorizedKeys; + boot.kernel.sysctl = { + "kernel.mm.transparent_hugepage.enabled" = "never"; + "net.core.somaxconn" = "512"; + }; + imports = [ ./nixos/configuration.nix (here + "/${name}.nix") @@ -166,10 +201,6 @@ rec { networking = { hostName = name; extraHosts = settings.clusterHosts; - # nameservers = [ masterAddress ]; - # dhcpcd.extraConfig = '' - # static domain_name_servers=${masterAddress} - # ''; firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; firewall.allowedTCPPorts = [ 80 443 111 ]; firewall.allowedUDPPorts = [ 111 24007 24008 ]; @@ -179,29 +210,33 @@ rec { ]; }; - apiserver = ip: name: self: + mkApiServer = host: self: { - deployment.targetHost = ip; + deployment.targetHost = host.address; require = [ - (baseNixos name) + (baseNixos host.name) kubeMaster ]; }; - worker = ip: name: self: + mkWorker = host: self: { - deployment.targetHost = ip; + deployment.targetHost = host.address; require = [ - (baseNixos name) + (baseNixos host.name) kubeWorker ]; }; - host = ip: name: self: + mkHost = host: self: { - deployment.targetHost = ip; + deployment.targetHost = host.address; require = [ - (baseNixos name) + (baseNixos host.name) ]; }; -} + + master = { "${settings.master.name}" = mkApiServer settings.master; }; +in + builtins.foldl' + (a: x: a // { "${x.name}" = mkWorker x; }) master settings.workers diff --git a/lib/nixos b/lib/nixos index 5fb88d7..4425906 160000 --- a/lib/nixos +++ b/lib/nixos @@ -1 +1 @@ -Subproject commit 5fb88d7ab6eb2236007aeeeab3eefa8ba1c39f36 +Subproject commit 4425906c65d264c7f8fcd46ced72458b273f57bc