Manage certificates using nix.

base/pki.nix

@@ -1,4 +1,3 @@
-# {pkgs, ...}:
 with import <nixpkgs> {};
 let
   ca-config = pkgs.writeText "ca-config.json" ''
@@ -37,6 +36,7 @@ let
       }
   }
   '';
+
   csr = args: pkgs.writeText "${args.cn}-cert.json" ''
     {
     "CN": "${args.cn}",
@@ -62,6 +62,7 @@ let
       buildInputs = [ pkgs.cfssl ];
     } '' cfssl genkey -initca ${ca-csr} | cfssljson -bare ca; \
          mkdir -p $out; cp *.pem $out'';
+
   ca_cert = "${ca}/ca.pem";
   ca_key = "${ca}/ca-key.pem";
 
@@ -70,58 +71,15 @@ let
     -config=${ca-config} -profile=${profile} ${name} | cfssljson -bare cert; \
     mkdir -p $out; cp *.pem $out
   '';
-  mkCert = cert:
+in
+  rec {
+    inherit ca_cert;
+    inherit ca_key;
+    inherit csr;
+
+    mkCert = cert:
     pkgs.runCommand "${cert.name}-cert" {
       buildInputs = [ pkgs.cfssl ];
     } (cfssl cert.csr cert.profile);
 
-  server-csr = csr {
+  }
-    cn = "kubernetes";
-    hosts = ''"kubernetes", "k8s0-0", "10.253.18.100"'';
-  };
-  server_cert = "${ca}/cert.pem";
-  server_key = "${ca}/cert-key.pem";
-
-  etcd0-csr = csr {
-    cn = "etcd0";
-    hosts = ''"etcd0", "10.253.18.100"'';
-  };
-  etcd0_cert = "${ca}/cert.pem";
-  etcd0_key = "${ca}/cert-key.pem";
-
-  etcd1-csr = csr {
-    cn = "etcd1";
-    hosts = ''"etcd1", "10.253.18.101"'';
-  };
-  etcd1_cert = "${ca}/cert.pem";
-  etcd1_key = "${ca}/cert-key.pem";
-
-  client-csr = csr {
-    cn = "client";
-    hosts = '''';
-  };
-  client_cert = "${ca}/cert.pem";
-  client_key = "${ca}/cert-key.pem";
-in
-rec {
-  server-cert = mkCert {
-      name = "kubernetes";
-      csr = server-csr;
-      profile = "server";
-  };
-  etcd0-cert = mkCert {
-      name = "etcd0";
-      csr = etcd0-csr;
-      profile = "peer";
-  };
-  etcd1-cert = mkCert {
-      name = "etcd1";
-      csr = etcd1-csr;
-      profile = "peer";
-  };
-  client-cert = mkCert {
-      name = "client";
-      csr = client-csr;
-      profile = "client";
-  };
-}

k8s.nix

@@ -1,27 +1,77 @@
+with import ./base/pki.nix;
 let
-  etcdConfig = name: {
+  server-cert = mkCert {
+      name = "kubernetes";
+      csr = csr {
+        cn = "kubernetes";
+        hosts = ''"kubernetes", "k8s0-0", "10.253.18.100"'';
+      };
+      profile = "server";
+  };
+
+  etcd0-cert = mkCert {
+    name = "etcd0";
+    csr = csr {
+      cn = "etcd0";
+      hosts = ''"etcd0", "10.253.18.100"'';
+    };
+    profile = "peer";
+  };
+
+  etcd1-cert = mkCert {
+    name = "etcd1";
+    csr = csr {
+      cn = "etcd1";
+      hosts = ''"etcd1", "10.253.18.101"'';
+    };
+    profile = "peer";
+  };
+
+  client-cert = mkCert {
+    name = "client";
+    csr = csr {
+      cn = "client";
+      hosts = '''';
+    };
+    profile = "client";
+  };
+
+  server_key = "${server-cert}/cert-key.pem";
+  server_cert = "${server-cert}/cert.pem";
+
+  etcd0_key = "${etcd0-cert}/cert-key.pem";
+  etcd0_cert = "${etcd0-cert}/cert.pem";
+
+  etcd1_key = "${etcd1-cert}/cert-key.pem";
+  etcd1_cert = "${etcd1-cert}/cert.pem";
+
+  client_key = "${client-cert}/cert-key.pem";
+  client_cert = "${client-cert}/cert.pem";
+
+  etcdCluster = [
+    "etcd0=https://etcd0:2380"
+    "etcd1=https://etcd1:2380"
+  ];
+
+  etcdConfig = etcd: {
     services.etcd = {
-      inherit name;
       enable = true;
       listenClientUrls = ["https://0.0.0.0:2379"];
       listenPeerUrls = ["https://0.0.0.0:2380"];
       peerClientCertAuth = true;
-      certFile = ./pki + "/${name}.pem";
+      keyFile  = "${etcd.key}";
-      keyFile  = ./pki + "/${name}-key.pem";
+      certFile = "${etcd.cert}";
-      trustedCaFile = ./pki/ca.pem;
+      trustedCaFile = "${ca_cert}";
-      advertiseClientUrls = [ "https://${name}:2379" ];
+      advertiseClientUrls = [ "https://${etcd.name}:2379" ];
-      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
+      initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ];
-      initialCluster = [
+      initialCluster = etcdCluster;
-        "etcd0=https://etcd0:2380"
+    };
-        "etcd1=https://etcd1:2380"
+    environment.variables = {
-      ];
+      ETCDCTL_KEY_FILE  = "${etcd.key}";
+      ETCDCTL_CERT_FILE = "${etcd.cert}";
+      ETCDCTL_CA_FILE   = "${ca_cert}";
+      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
     };
-    # environment.variables = {
-    #   ETCDCTL_CERT_FILE = ./pki + "/${name}.pem";
-    #   ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem";
-    #   ETCDCTL_CA_FILE = ./pki/ca.pem;
-    #   ETCDCTL_PEERS = "https://127.0.0.1:2379";
-    # };
     networking.firewall.allowedTCPPorts = [ 2379 2380 ];
   };
 
@@ -32,9 +82,9 @@ let
       iface = "enp2s0";
       etcd = {
         endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
-        caFile   = ./pki/ca.pem;
+        caFile   = "${ca_cert}";
-        certFile = ./pki/client.pem;
+        keyFile  = "${client_key}";
-        keyFile  = ./pki/client-key.pem;
+        certFile = "${client_cert}";
       };
     };
   };
@@ -43,16 +93,19 @@ let
     require = [ flannelConfig ];
     networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
     networking.firewall.allowedTCPPorts = [ 10250 ];
-    systemd.services.docker.after = [ "flannel.service" ];
+    systemd.services.docker = {
-    systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
+      after = [ "flannel.service" ];
-    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
+      serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
-    # services.kubernetes.verbose = true;
+    };
+    virtualisation.docker.extraOptions =
+      "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
     services.kubernetes.etcd = {
       servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
-      caFile   = ./pki/ca.pem;
+      caFile   = "${ca_cert}";
-      certFile = ./pki/client.pem;
+      keyFile  = "${client_key}";
-      keyFile  = ./pki/client-key.pem;
+      certFile = "${client_cert}";
     };
+    # services.kubernetes.verbose = true;
   };
 
   kubeNode = {
@@ -60,13 +113,14 @@ let
       roles = [ "node" ];
       kubeconfig = {
         server = "https://kubernetes:443";
-        caFile   = ./pki/ca.pem;
+        caFile   = "${ca_cert}";
-        certFile = ./pki/client.pem;
+        keyFile  = "${client_key}";
-        keyFile  = ./pki/client-key.pem;
+        certFile = "${client_cert}";
       };
       kubelet = {
-        tlsCertFile = ./pki/client.pem;
+        tlsKeyFile  = "${client_key}";
-        tlsKeyFile  = ./pki/client-key.pem;
+        tlsCertFile = "${client_cert}";
+        extraOpts = "--client-ca-file=${ca_cert}";
         networkPlugin = null;
         clusterDns = "kubernetes";
       };
@@ -83,16 +137,16 @@ let
       apiserver = {
         publicAddress = "0.0.0.0";
         address = "0.0.0.0";
-        clientCaFile = ./pki/ca.pem;
+        clientCaFile = "${ca_cert}";
-        tlsCertFile  = ./pki/server.pem;
+        tlsKeyFile   = "${server_key}";
-        tlsKeyFile   = ./pki/server-key.pem;
+        tlsCertFile  = "${server_cert}";
-        # kubeletClientCaFile   = ./pki/ca.pem;
+        # kubeletClientCaFile   = "${ca_cert}";
-        # kubeletClientCertFile = ./pki/client.pem;
+        # kubeletClientKeyFile  = "${server_key}";
-        # kubeletClientKeyFile  = ./pki/client-key.pem;
+        # kubeletClientCertFile = "${server_cert}";
       };
       scheduler.leaderElect = true;
       controllerManager.leaderElect = true;
-      controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
+      controllerManager.serviceAccountKeyFile = "${server_key}";
     };
 
     networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
@@ -114,8 +168,12 @@ in
   k8s0-0 = { config, lib, pkgs, ... }:
     let
       host = "k8s0-0";
-      etcd = etcdConfig "etcd0";
       base = baseConfig host;
+      etcd = etcdConfig {
+        name = "etcd0";
+        key = etcd0_key;
+        cert = etcd0_cert;
+      };
     in
     {
       deployment.targetHost = "10.253.18.100";
@@ -125,8 +183,12 @@ in
   k8s0-1 = { config, lib, pkgs, ... }:
     let
       host = "k8s0-1";
-      etcd = etcdConfig "etcd1";
       base = baseConfig host;
+      etcd = etcdConfig {
+        name = "etcd1";
+        key = etcd1_key;
+        cert = etcd1_cert;
+      };
     in
     {
       deployment.targetHost = "10.253.18.101";

pki/ca-config.json

@@ -1,34 +0,0 @@
-{
-    "signing": {
-        "default": {
-            "expiry": "43800h"
-        },
-        "profiles": {
-            "server": {
-                "expiry": "43800h",
-                "usages": [
-                    "signing",
-                "key encipherment",
-                "server auth"
-                    ]
-            },
-            "client": {
-                "expiry": "43800h",
-                "usages": [
-                    "signing",
-                "key encipherment",
-                "client auth"
-                    ]
-            },
-            "peer": {
-                "expiry": "43800h",
-                "usages": [
-                    "signing",
-                "key encipherment",
-                "server auth",
-                "client auth"
-                    ]
-            }
-        }
-    }
-}

pki/ca.json

@@ -1,16 +0,0 @@
-{
-    "CN": "k8s0",
-    "key": {
-        "algo": "rsa",
-        "size": 2048
-    },
-    "names": [
-        {
-            "C": "NO",
-            "L": "Tromsø",
-            "O": "Serit IT Partner Tromsø AS",
-            "OU": "",
-            "ST": ""
-        }
-    ]
-}

pki/client.json

@@ -1,17 +0,0 @@
-{
-    "CN": "client",
-    "hosts": [ "" ],
-    "key": {
-        "algo": "rsa",
-        "size": 2048
-    },
-    "names": [
-        {
-            "C": "NO",
-            "L": "Tromsø",
-            "O": "Serit IT Partner Tromsø AS",
-            "OU": "",
-            "ST": ""
-        }
-    ]
-}

pki/etcd.json

@@ -1,20 +0,0 @@
-{
-    "CN": "@host@",
-    "hosts": [
-        "@host@",
-        "@ip"
-    ],
-    "key": {
-        "algo": "rsa",
-        "size": 2048
-    },
-    "names": [
-        {
-            "C": "NO",
-            "L": "Tromsø",
-            "O": "Serit IT Partner Tromsø AS",
-            "OU": "",
-            "ST": ""
-        }
-    ]
-}

pki/mkcerts.sh

@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-etcd="etcd0,100 etcd1,101"
-
-cacert () {
-    cfssl genkey -initca ca.json | cfssljson -bare ca
-}
-
-servercert () {
-    cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-        -config=ca-config.json -profile=server server.json \
-        | cfssljson -bare server
-}
-
-etcdcert () {
-    host=$1
-    ip=$2
-    sed "s/@host@/$host/g; s/@ip@/$ip/g;" etcd.json \
-        | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-        -config=ca-config.json -profile=peer - \
-        | cfssljson -bare $host
-}
-
-clientcert () {
-    cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \
-        -config=ca-config.json -profile=client client.json \
-        | cfssljson -bare client
-}
-
-mketcdcerts () {
-    for i in $etcd; do
-        IFS=","
-        set -- $i
-        etcdcert $1 10.253.18.$2
-    done
-}
-
-case $1 in
-    all)
-        cacert
-        servercert
-        mketcdcerts
-        clientcert
-        ;;
-    client)
-        clientcert
-        ;;
-    api)
-        servercert
-        ;;
-    etcd)
-        mketcdcerts
-        ;;
-    *)
-        echo "usege: mkcerts.sh (all|client|api|etcd)"
-        exit 1
-        ;;
-esac

pki/server.json

@@ -1,21 +0,0 @@
-{
-    "CN": "server",
-    "hosts": [
-        "k8s0-0",
-        "kubernetes",
-        "10.253.18.100"
-    ],
-    "key": {
-        "algo": "rsa",
-        "size": 2048
-    },
-    "names": [
-        {
-            "C": "NO",
-            "L": "Tromsø",
-            "O": "Serit IT Partner Tromsø AS",
-            "OU": "",
-            "ST": ""
-        }
-    ]
-}