From c21abb5a5f79bfc19e5708e6124e12f6402b944d Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Mon, 10 Jul 2017 13:40:09 +0200 Subject: [PATCH] Lots of small fixes. Now it works! --- k8s.nix | 113 +++++++++++++++++++++++++-------------------- pki/apiserver.json | 1 + pki/client.json | 5 ++ pki/mkcerts.sh | 39 ++++++++-------- pki/pki.nix | 11 +++++ 5 files changed, 98 insertions(+), 71 deletions(-) create mode 100644 pki/pki.nix diff --git a/k8s.nix b/k8s.nix index 2b1f3cf..f669399 100644 --- a/k8s.nix +++ b/k8s.nix @@ -2,80 +2,86 @@ let etcdConfig = name: { services.etcd = { inherit name; - advertiseClientUrls = [ "https://${name}:2379" ]; - initialAdvertisePeerUrls = [ "https://${name}:2380" ]; enable = true; + listenClientUrls = ["https://0.0.0.0:2379"]; + listenPeerUrls = ["https://0.0.0.0:2380"]; + peerClientCertAuth = true; certFile = ./pki/etcd.pem; keyFile = ./pki/etcd-key.pem; trustedCaFile = ./pki/ca.pem; - peerClientCertAuth = true; - listenClientUrls = ["https://0.0.0.0:2379"]; - listenPeerUrls = ["https://0.0.0.0:2380"]; + advertiseClientUrls = [ "https://${name}:2379" ]; + initialAdvertisePeerUrls = [ "https://${name}:2380" ]; initialCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ]; - # environment.variables = { - # ETCDCTL_CERT_FILE = ./pki/client.pem; - # ETCDCTL_KEY_FILE = ./pki/client-key.pem; - # ETCDCTL_CA_FILE = ./pki/ca.pem; - # ETCDCTL_PEERS = "https://127.0.0.1:2379"; - # }; }; + # environment.variables = { + # ETCDCTL_CERT_FILE = ./pki + "/${name}.pem"; + # ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem"; + # ETCDCTL_CA_FILE = ./pki/ca.pem; + # ETCDCTL_PEERS = "https://127.0.0.1:2379"; + # }; networking.firewall.allowedTCPPorts = [ 2379 2380 ]; }; - flannelConfig = { + flannelConfig = node: { services.flannel = { enable = true; network = "10.10.0.0/16"; - iface = "enp0s3"; + iface = "enp2s0"; etcd = { - endpoints = ["https://etcd0:2379" "https://etcd1:2379" ]; - certFile = ./pki/client.pem; - keyFile = ./pki/client-key.pem; + endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ]; + certFile = ./pki + "/${node}.pem"; + keyFile = ./pki + "/${node}-key.pem"; caFile = ./pki/ca.pem; }; }; }; - kubeNode = { + etcdClient = node:{ + servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; + certFile = ./pki + "/${node}.pem"; + keyFile = ./pki + "/${node}-key.pem"; + caFile = ./pki/ca.pem; + }; + + kubeConfig = node: { + require = [ (flannelConfig node) ]; + networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN + networking.firewall.allowedTCPPorts = [ 10250 ]; + systemd.services.docker.after = [ "flannel.service" ]; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; + virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; + # services.kubernetes.verbose = true; + }; + + kubeNode = doConfig: node: { + require = if doConfig then [ (kubeConfig node) ] else []; services.kubernetes = { - # verbose = true; roles = [ "node" ]; kubeconfig = { server = "https://kubernetes:443"; caFile = ./pki/ca.pem; - certFile = ./pki/client.pem; - keyFile = ./pki/client-key.pem; + certFile = ./pki + "/${node}.pem"; + keyFile = ./pki + "/${node}-key.pem"; }; - etcd = { - servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; - certFile = ./pki/client.pem; - keyFile = ./pki/client-key.pem; - caFile = ./pki/ca.pem; + kubelet = { + tlsCertFile = ./pki + "/${node}.pem"; + tlsKeyFile = ./pki + "/${node}-key.pem"; + networkPlugin = null; + clusterDns = "10.253.18.100"; }; - # kubelet.clusterDns = "10.10.1.1"; + etcd = if doConfig then (etcdClient node) else {}; }; - - networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN - networking.firewall.allowedTCPPorts = [ 10250 ]; - networking.extraHosts = '' - 10.253.18.100 etcd0 kubernetes - 10.253.18.101 etcd1 - ''; - - systemd.services.docker.after = [ "flannel.service" ]; - systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; }; - kubeMaster = { + kubeMaster = node: { + require = [ (kubeConfig node) (kubeNode false node)]; services.dockerRegistry = { enable = true; listenAddress = "0.0.0.0"; }; - services.kubernetes = { roles = [ "master" ]; apiserver = { @@ -84,10 +90,11 @@ let clientCaFile = ./pki/ca.pem; tlsCertFile = ./pki/apiserver.pem; tlsKeyFile = ./pki/apiserver-key.pem; - kubeletClientCaFile = ./pki/ca.pem; - kubeletClientCertFile = ./pki/client.pem; - kubeletClientKeyFile = ./pki/client-key.pem; + # kubeletClientCaFile = ./pki/ca.pem; + # kubeletClientCertFile = ./pki + "/${node}.pem"; + # kubeletClientKeyFile = ./pki + "/${node}-key.pem"; }; + etcd = (etcdClient node); scheduler.leaderElect = true; controllerManager.leaderElect = true; controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem; @@ -98,9 +105,13 @@ let systemd.services.flannel.after = [ "etcd.service" ]; }; - baseConfig = name: { - networking.hostName = name; - imports = [ "./hw/${name}.nix" ./base/configuration.nix ]; + baseConfig = node: { + networking.hostName = node; + imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; + networking.extraHosts = '' + 10.253.18.100 etcd0 kubernetes + 10.253.18.101 etcd1 + ''; }; in { @@ -108,31 +119,31 @@ in let etcd = etcdConfig "etcd0"; base = baseConfig "k8s0-0"; + master = kubeMaster "k8s0-0"; in { deployment.targetHost = "10.253.18.100"; - require = [ base etcd flannelConfig ]; - # require = [ base etcd flannelConfig kubeMaster kubeNode ]; + require = [ base etcd master ]; }; k8s0-1 = { config, lib, pkgs, ... }: let etcd = etcdConfig "etcd1"; base = baseConfig "k8s0-1"; + node = kubeNode true "k8s0-1"; in { deployment.targetHost = "10.253.18.101"; - require = [ base etcd flannelConfig ]; - # require = [ base etcd flannelConfig kubeNode ]; + require = [ base etcd node ]; }; k8s0-2 = { config, lib, pkgs, ... }: let base = baseConfig "k8s0-2"; + node = kubeNode true "k8s0-2"; in { deployment.targetHost = "10.253.18.102"; - require = [ base flannelConfig ]; - # require = [ base flannelConfig kubeNode ]; + require = [ base node ]; }; } diff --git a/pki/apiserver.json b/pki/apiserver.json index d1da0e7..22e50df 100644 --- a/pki/apiserver.json +++ b/pki/apiserver.json @@ -1,6 +1,7 @@ { "hosts": [ "k8s0-0", + "kubernetes", "10.253.18.100" ], "key": { diff --git a/pki/client.json b/pki/client.json index 5a28665..c4fb879 100644 --- a/pki/client.json +++ b/pki/client.json @@ -1,4 +1,9 @@ { + "CN": "@host@", + "hosts": [ + "@host@", + "@ip@" + ], "key": { "algo": "rsa", "size": 2048 diff --git a/pki/mkcerts.sh b/pki/mkcerts.sh index 8b66987..23a8fb5 100755 --- a/pki/mkcerts.sh +++ b/pki/mkcerts.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102" +hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102" mkcacert () { cfssl genkey -initca ca.json | cfssljson -bare ca @@ -16,27 +16,26 @@ mketcdcert () { | cfssljson -bare etcd } +# mkclientcert () { +# cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \ +# | cfssljson -bare client +# } + mkclientcert () { - cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \ - | cfssljson -bare client + host=$1 + ip=$2 + sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \ + | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \ + | cfssljson -bare $host } -# mkclientcert () { -# host=$1 -# ip=$2 - -# sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \ -# | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \ -# | cfssljson -bare $host -# } - -# mkclientcerts () { -# for i in $hosts; do -# IFS="," -# set -- $i -# mkclientcert $1 10.253.18.$2 -# done -# } +mkclientcerts () { + for i in $hosts; do + IFS="," + set -- $i + mkclientcert $1 10.253.18.$2 + done +} case $1 in all) @@ -46,7 +45,7 @@ case $1 in mkclientcert ;; client) - mkclientcert + mkclientcerts ;; api) mkapicert diff --git a/pki/pki.nix b/pki/pki.nix new file mode 100644 index 0000000..0e0fdb7 --- /dev/null +++ b/pki/pki.nix @@ -0,0 +1,11 @@ +{pkgs, ...}: +let + makeCert = name: + pkgs.runCommand name { + buildInputs = [ pkgs.cfssl ]; + } ''cfssl gencert -ca ca.pem -ca-key ca-key.pem ${name}.json \ + | cfssljson -bare ${name}''; +in +{ + ca_key +}