diff --git a/lib/base.nix b/lib/base.nix new file mode 100644 index 0000000..627bf15 --- /dev/null +++ b/lib/base.nix @@ -0,0 +1,63 @@ +{ pkgs, lib, settings, here ? "", ...}: +with lib; +rec { + pki = import ./pki.nix { inherit pkgs; ca = settings.initca; }; + + baseNixos = name: { + users.extraUsers.admin.openssh.authorizedKeys.keys = + settings.adminAuthorizedKeys; + + boot.kernel.sysctl = { + "kernel.mm.transparent_hugepage.enabled" = "never"; + "net.core.somaxconn" = "512"; + }; + + imports = [ + ./nixos/configuration.nix + (here + "/${name}.nix") + ]; + + security.pki.certificateFiles = [ + pki.ca.cert + ]; + + networking = { + hostName = name; + extraHosts = settings.clusterHosts; + firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; + firewall.allowedTCPPorts = [ 80 443 111 ]; + firewall.allowedUDPPorts = [ 111 24007 24008 ]; + }; + environment.systemPackages = with pkgs; [ + nfs-utils + ]; + }; + + hostCerts = + builtins.foldl' + (a: x: a // { ${x.name} = pki.gencert { + cn = x.name; + ca = x.ca; + o = settings.clusterName; + }; + }) {} settings.hosts; + + + mkHost = host: self: + { + deployment.targetHost = host.address; + require = [ + (baseNixos host.name) + ]; + }; + + baseDeployment = attrs: + let + hosts = + builtins.foldl' + (a: x: a // { ${x.name} = mkHost x _; }) {} settings.hosts; + hosts' = lib.recursiveUpdate hosts attrs; + names = builtins.attrNames hosts; + in + builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names; +} diff --git a/lib/initca.nix b/lib/initca.nix index a679e39..8fd4e21 100644 --- a/lib/initca.nix +++ b/lib/initca.nix @@ -1,18 +1,18 @@ -{ pkgs ? import {}, initca ? "", ...}: +{ pkgs ? import {}, ca ? "", name ? "ca", ...}: with pkgs; let - initca' = + ca' = let - ca_csr = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON { + ca_csr = pkgs.writeText "${name}-csr.json" (builtins.toJSON { key = { algo = "rsa"; size = 2048; }; names = [ { - CN = "kubernetes-cluster-ca"; + CN = "${name}"; O = "NixOS"; - OU = "services.kubernetes.pki.caSpec"; + OU = "${name}.pki.caSpec"; L = "generated"; } ]; @@ -22,12 +22,12 @@ let buildInputs = [ pkgs.cfssl ]; } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \ mkdir -p $out; cp *.pem $out''; - ca = if initca != "" then initca else initca'; + initca = if ca != "" then ca else ca'; in # make ca derivation sha depend on initca cfssl output pkgs.stdenv.mkDerivation { - name = "ca"; - src = ca; + inherit name; + src = initca; buildCommand = '' mkdir -p $out; cp -r $src/* $out diff --git a/lib/k8s.nix b/lib/k8s.nix index dad3110..bb882c3 100644 --- a/lib/k8s.nix +++ b/lib/k8s.nix @@ -1,11 +1,9 @@ { pkgs, lib, settings, here ? "", ...}: +with import ./base.nix { inherit pkgs lib settings here; }; with lib; let apiserverAddress = "https://${masterAddress}:4443"; masterAddress = settings.master.address; - initca = settings.initca; - - cluster-ca = import ./initca.nix { inherit pgks initca; }; cfssl-apitoken = let @@ -47,7 +45,7 @@ let export bash="${pkgs.bash}" export apiserver="${settings.master.name}" export apiserverAddress="${settings.master.address}" - export initca="${initca}" + export initca="${pki.initca}" export cluster="${clusterName}" export fileserver="${fileserver}" export acme_email="${acme_email}" @@ -89,8 +87,8 @@ let ''; kubeMaster = { - services.cfssl.ca = "${cluster-ca}/ca.pem"; - services.cfssl.caKey = "${cluster-ca}/ca-key.pem"; + services.cfssl.ca = pki.ca.cert; + services.cfssl.caKey = pki.ca.key; services.kubernetes = { roles = [ "master" ]; inherit apiserverAddress; @@ -98,7 +96,7 @@ let clusterCidr = settings.cidr; pki.genCfsslCACert = false; pki.genCfsslAPIToken = false; - pki.caCertPathPrefix = "${cluster-ca}/ca"; + pki.caCertPathPrefix = "${pki.initca}"; kubelet = { clusterDomain = "${settings.clusterName}.local"; @@ -110,7 +108,7 @@ let allowPrivileged = true; securePort = 4443; insecurePort = 8080; - extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem"; + extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}"; # verbosity = 4; }; @@ -195,42 +193,6 @@ let }; }; - baseNixos = name: { - users.extraUsers.admin.openssh.authorizedKeys.keys = - settings.adminAuthorizedKeys; - - boot.kernel.sysctl = { - "kernel.mm.transparent_hugepage.enabled" = "never"; - "net.core.somaxconn" = "512"; - }; - - imports = [ - ./nixos/configuration.nix - (here + "/${name}.nix") - ]; - security.pki.certificateFiles = [ - "${cluster-ca}/ca.pem" - ]; - # services.glusterfs = { - # enable = true; - # # tlsSettings = { - # # caCert = certs.ca.caFile; - # # tlsKeyPath = certs.self.keyFile; - # # tlsPem = certs.self.certFile; - # }; - # }; - networking = { - hostName = name; - extraHosts = settings.clusterHosts; - firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; - firewall.allowedTCPPorts = [ 80 443 111 ]; - firewall.allowedUDPPorts = [ 111 24007 24008 ]; - }; - environment.systemPackages = with pkgs; [ - nfs-utils - ]; - }; - mkApiServer = host: self: { deployment.targetHost = host.address; @@ -249,15 +211,9 @@ let ]; }; - mkHost = host: self: - { - deployment.targetHost = host.address; - require = [ - (baseNixos host.name) - ]; - }; - master = { "${settings.master.name}" = mkApiServer settings.master; }; + + deployment = builtins.foldl' (a: x: + a // { "${x.name}" = mkWorker x; }) master settings.workers; in - builtins.foldl' - (a: x: a // { "${x.name}" = mkWorker x; }) master settings.workers + deployment diff --git a/lib/nixos b/lib/nixos index 4425906..9d8148f 160000 --- a/lib/nixos +++ b/lib/nixos @@ -1 +1 @@ -Subproject commit 4425906c65d264c7f8fcd46ced72458b273f57bc +Subproject commit 9d8148f2727643db9dcc31db2a0db447098d0755 diff --git a/lib/pki.nix b/lib/pki.nix index 3e5c06c..d24a039 100644 --- a/lib/pki.nix +++ b/lib/pki.nix @@ -1,5 +1,12 @@ -{ pkgs ? import {} }: +{ pkgs, ca ? "" }: let + initca = import ./initca.nix { inherit pkgs ca; }; + + ca' = { + key = "${initca}/ca-key.pem"; + cert = "${initca}/ca.pem"; + }; + ca-config = pkgs.writeText "ca-config.json" '' { "signing": { @@ -7,7 +14,7 @@ let "expiry": "8760h" }, "profiles": { - "kubernetes": { + "default": { "usages": [ "signing", "key encipherment", @@ -21,135 +28,55 @@ let } ''; - csr = o: { - key = { - algo = "rsa"; - size = 2048; - }; - names = [ - { - CN = "kubernetes-cluster-ca"; - O = "${o}"; - OU = "services.kubernetes.pki.caSpec"; - L = "generated"; - } - ]; - }; - - gencsr = args: pkgs.writeText "${args.name}-csr.json" (builtins.toJSON { - CN = "${args.cn}"; - hosts = [ "${args.hosts}" ]; - } // csr args.o - ); - - initca' = + gencsr = args: let - ca_csr = pkgs.writeText "kube-pki-cacert-csr.json" ( - builtins.toJSON (csr "NixOS") - ); + csr = { + CN = "${args.cn}"; + key = { + algo = "rsa"; + size = 2048; + }; + names = [ + { + CN = "${args.cn}"; + O = "${args.o}"; + OU = "${args.cn}.${args.o}.pki.caSpec"; + L = "generated"; + } + ]; + hosts = [ "${args.hosts}" ]; + }; in - pkgs.runCommand "initca" { - buildInputs = [ pkgs.cfssl ]; - } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \ - mkdir -p $out; cp *.pem $out''; - - # make ca derivation sha depend on initca cfssl output - initca = pkgs.stdenv.mkDerivation { - name = "ca"; - src = initca'; - buildCommand = '' - mkdir -p $out; - cp -r $src/* $out - ''; - }; - - ca = { - key = "${initca}/ca-key.pem"; - cert = "${initca}/ca.pem"; - }; - - cfssl = conf: '' - cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \ - -config=${ca-config} -profile=kubernetes ${conf.csr} | \ - cfssljson -bare cert; \ - mkdir -p $out; cp *.pem $out - ''; - - gencert = conf: - let crt = - pkgs.runCommand "${conf.name}" { - buildInputs = [ pkgs.cfssl ]; - } (cfssl conf); + pkgs.writeText "${args.cn}-csr.json" (builtins.toJSON csr); +in +# Example usage: +# +# gencert { cn = "test"; ca = ca; o = "test; }; +# +rec { + inherit initca; + ca = ca'; + gencert = attrs: + let + conf = { + cn = attrs.cn; + ca = attrs.ca; + csr = gencsr { cn = attrs.cn; o = attrs.o; hosts = ""; }; + }; + cfssl = conf: + '' + cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \ + -config=${ca-config} -profile=default ${conf.csr} | \ + cfssljson -bare cert; \ + mkdir -p $out; cp *.pem $out + ''; + crt = + pkgs.runCommand "${attrs.cn}" { + buildInputs = [ pkgs.cfssl ]; + } (cfssl conf); in { key = "${crt}/cert-key.pem"; cert = "${crt}/cert.pem"; }; - - trust = name: hosts: - let - hosts' = "\"${name}\", " + hosts; - in gencert rec { - inherit name; - csr = gencsr { - inherit name; - hosts = hosts'; - cn = name; - o = name; - }; - }; -in -{ - inherit ca; - - admin = gencert rec { - name = "admin"; - csr = gencsr { - inherit name; - cn = "admin"; - o = "system:masters"; - hosts = ""; - }; - }; - - apiserver = hosts: - gencert rec { - name = "kubernetes"; - csr = gencsr { - inherit name hosts; - cn = "kubernetes"; - o = "kubernetes"; - }; - }; - - etcd = hosts: gencert rec { - name = "etcd"; - csr = gencsr { - inherit name hosts; - cn = "etcd"; - o = "kubernetes"; - }; - }; - - kube-proxy = gencert rec { - name = "kube-proxy"; - csr = gencsr { - inherit name; - cn = "system:kube-proxy"; - o = "system:node-proxier"; - hosts = ""; - }; - }; - - worker = instance: - gencert rec { - name = instance.name; - csr = gencsr { - inherit name; - cn = "system:node:${instance.name}"; - o = "system:nodes"; - hosts = ''"${instance.name}","${instance.ip}"''; - }; - }; } -