Migration to new setup

bin/deploy.sh

@@ -19,4 +19,4 @@ echo "--- Updating deployment"
 nixops modify -d $1 $1/deployment.nix
 
 echo "--- Deploying $1"
-nixops deploy -d $*
+nixops deploy -d $* --allow-reboot

bin/teardown.sh

@@ -18,5 +18,5 @@ f=.$d.$$
 # rm $f
 
 nixops reboot -d $d
-nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"
+nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
 

kube0/deployment.nix

@@ -3,7 +3,7 @@ let
   certs = pkgs.callPackage ./certs.nix {};
   pki = pkgs.callPackage ../lib/pki.nix {};
   cluster = callPackage ../lib/k8s.nix {
-    masterNode = "10.253.18.100";
+    masterAddress = "10.253.18.100";
     etcdNodes = [ "etcd0" "etcd1" "etcd2" ];
     clusterHosts = ''
           10.253.18.100 k0-0 etcd0 kubernetes
@@ -15,7 +15,7 @@ let
           10.253.18.106 fs0-0 fs0-0.local
           10.1.2.164    fs0-1 fs0-1.local
           10.253.18.100 fs0-2 fs0-2.local
-          10.253.18.100 itp-registry registry.itpartner.no minio.itpartner.no
+          10.253.18.100 itp-registry
           10.253.18.100 nuget.itpartner.no
           10.253.18.109 k1-0
     '';
@@ -35,7 +35,10 @@ let
   };
 in
 {
-  k0-0 = { ... }: {
+  k0-0 = { ... }:
+    let
+      apiserver = pki.toSet certs.apiserver;
+    in {
     require = [ (cluster.apiserver "10.253.18.100" "k0-0" "etcd0") ];
     boot.kernelModules = [
       "dm_snapshot"
@@ -51,11 +54,20 @@ in
       device = "fs0-0:docker-registry";
       fsType = "nfs4";
     };
+    services.dockerRegistry = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+      enableDelete = true;
+      enableGarbageCollect = true;
+      extraConfig = {
+        REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver.cert}";
+        REGISTRY_HTTP_TLS_KEY = "${apiserver.key}";
+      };
+    };
     environment.systemPackages = [ pkgs.lvm2 ];
     networking.extraHosts = ''
       10.253.18.100 itp-registry itp-registry.local
       10.253.18.100 helm-registry helm-registry.local
-      10.253.18.100 gitlab.itpartner.no registry.itpartner.no minio.itpartner.no
       10.253.18.100 nuget.local
       10.253.18.100 dashboard.k0.local
       10.253.18.100 gitlab.k0.local

kube1/certs.nix

@@ -3,7 +3,7 @@ let
   pki = pkgs.callPackage ../lib/pki.nix {};
 in
 {
-    initca = pki.initca;
+    # initca = pki.initca;
     ca = pki.ca;
     apiserver = pki.apiserver ''
       "10.253.18.109",

kube1/configure.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+helm init
+echo "Waiting for tiller"
+sleep 30
+
+
+helm install --namespace kube-system --name ifs1 -f ifs1.yaml stable/nfs-client-provisioner
+

kube1/deploy.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+id=kube1
+
+# if [ $# = 0 ]; then
+#     echo "usage: deploy.sh name ..."
+#     exit 1
+# fi
+
+if [ ! -f ./deployment.nix ]; then
+    echo "error: ./ does not contain a deployment"
+    exit 1
+fi
+
+# mkdir -p $1/gcroots
+
+# echo "--- Securing certifiates"
+# nix-build -o $1/gcroots/certs $1/build.nix
+
+echo "--- Updating deployment"
+nixops modify -d $id ./deployment.nix
+
+echo "--- Deploying $id"
+nixops deploy -d $id --allow-reboot
+

kube1/deployment.nix

@@ -1,38 +1,30 @@
 with import <nixpkgs> {};
 let
-  certs = pkgs.callPackage ./certs.nix {};
+  settings = rec {
-  pki = pkgs.callPackage ../lib/pki.nix {};
+    master = "k1-0";
-  cluster = callPackage ../lib/k8s.nix {
+    workers = [ "k1-1" "k1-2" ];
-    masterNode = "10.253.18.109";
+    masterAddress = "10.253.18.109";
-    etcdNodes = [ "etcd0" "etcd1" ];
+    apiserverAddress = "https://${masterAddress}:8443";
     clusterHosts = ''
-      10.253.18.109 k1-0 etcd0 kubernetes fs0-2
+      10.253.18.109 k1-0 kubernetes fs0-2
-      10.253.18.110 k1-1 etcd1
+      10.253.18.110 k1-1
       10.253.18.111 k1-2
       10.253.18.106 fs0-0
       10.1.2.164    fs0-1
       10.253.18.100 k0-0
       10.253.18.100 gitlab.itpartner.no registry.itpartner.no minio.itpartner.no
-      10.253.18.109 gitlab.k1.local registry.k1.local minio.k1.local
-      10.253.18.100 itp-registry itp-registry.local
     '';
-    certs = {
-      ca = certs.ca;
-      apiserver = pki.toSet certs.apiserver;
-      kube-proxy = pki.toSet certs.kube-proxy;
-      admin = pki.toSet certs.admin;
-      etcd = pki.toSet certs.etcd;
-      k1-0 = pki.toSet certs.k1-0;
-      k1-1 = pki.toSet certs.k1-1;
-      k1-2 = pki.toSet certs.k1-2;
-    };
   };
+  cluster = callPackage ./k8s.nix { inherit settings; };
 in
 {
-  k1-0 = { ... }:
+  # k1-0 = cluster.host "10.253.18.109" "k1-0";
+  # k1-1 = cluster.host "10.253.18.110" "k1-1";
+  # k1-2 = cluster.host "10.253.18.111" "k1-2";
+  k1-0 = self:
   {
-    require = [ (cluster.apiserver "10.253.18.109" "k1-0" "etcd0") ];
+    require = [ (cluster.apiserver "10.253.18.109" "k1-0") ];
   };
-  k1-1 = cluster.server "10.253.18.110" "k1-1" "etcd1";
+  k1-1 = cluster.worker "10.253.18.110" "k1-1";
   k1-2 = cluster.worker "10.253.18.111" "k1-2";
 }

kube1/k8s.nix

@@ -0,0 +1,136 @@
+{ pkgs, lib, settings, ...}:
+with lib;
+let
+  cluster-ca = pkgs.stdenv.mkDerivation {
+      name = "cluster-ca";
+      src = ./ca;
+      buildCommand = ''
+        mkdir -p $out
+        cp $src/* $out
+      '';
+  };
+  nixos-kubernetes-join-nodes = workers:
+    let
+      wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
+    in
+    pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
+      #!/bin/sh
+      set -e
+      token=$(cat /var/lib/cfssl/apitoken.secret)
+      for i in ${wrk}; do
+         ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
+      done
+    '';
+  cidr = "10.10.0.0/16";
+in
+rec {
+  kubeMaster = {
+    services.cfssl.ca = "${cluster-ca}/ca.pem";
+    services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
+    services.kubernetes = {
+      roles = [ "master" ];
+      masterAddress = settings.master;
+      apiserverAddress = settings.apiserverAddress;
+      clusterCidr = cidr;
+      kubelet.unschedulable = false;
+      pki.genCfsslCACert = false;
+      pki.caCertPathPrefix = "${cluster-ca}/ca";
+      apiserver = {
+        advertiseAddress = settings.masterAddress;
+        authorizationMode = [ "Node" "RBAC" ];
+        securePort = 8443;
+        insecurePort = 8080;
+        extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
+      };
+      addons = {
+        dns = {
+          enable = true;
+          # clusterDomain = "local";
+          reconcileMode = "EnsureExists";
+        };
+      };
+    };
+    networking.firewall = {
+      allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
+      allowedUDPPorts = [ 53 4053 ];
+    };
+    environment.systemPackages = [
+      pkgs.kubernetes-helm
+      (nixos-kubernetes-join-nodes settings.workers)
+    ];
+  };
+
+  kubeWorker = {
+    services.kubernetes = rec {
+      roles = [ "node" ];
+      clusterCidr = cidr;
+      masterAddress = settings.master;
+      apiserverAddress = settings.apiserverAddress;
+    };
+    networking = {
+      firewall = {
+        enable = true;
+        allowedTCPPorts = [ 4194 10250 ];
+        allowedUDPPorts = [ 53 ];
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
+      };
+    };
+    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
+    virtualisation.docker.autoPrune.enable = true;
+  };
+
+  baseNixos = name: {
+    imports = [
+      (../nixos/hardware-configuration + "/${name}.nix")
+      ../nixos/configuration.nix
+    ];
+    security.pki.certificateFiles = [
+      "${cluster-ca}/ca.pem"
+    ];
+    # services.glusterfs = {
+    #   enable = true;
+    #   # tlsSettings = {
+    #     # caCert = certs.ca.caFile;
+    #     # tlsKeyPath = certs.self.keyFile;
+    #     # tlsPem = certs.self.certFile;
+    #   };
+    # };
+    networking = {
+      hostName = name;
+      extraHosts = settings.clusterHosts;
+      # nameservers = [ masterAddress ];
+      # dhcpcd.extraConfig = ''
+      #  static domain_name_servers=${masterAddress}
+      # '';
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 80 443 111 ];
+      firewall.allowedUDPPorts = [ 111 24007 24008 ];
+    };
+  };
+
+  apiserver = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+        kubeMaster
+      ];
+    };
+
+  worker = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+        kubeWorker
+      ];
+    };
+
+  host = ip: name: self:
+    {
+      deployment.targetHost = ip;
+      require = [
+        (baseNixos name)
+      ];
+    };
+}

lib/initca.nix

@@ -0,0 +1,33 @@
+with import <nixpkgs> {};
+let
+  initca' =
+    let
+      ca_csr = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
+          key = {
+              algo = "rsa";
+              size = 2048;
+          };
+          names = [
+            {
+              CN = "kubernetes-cluster-ca";
+              O = "NixOS";
+              OU = "services.kubernetes.pki.caSpec";
+              L = "generated";
+            }
+          ];
+        });
+    in
+      pkgs.runCommand "initca" {
+        buildInputs = [ pkgs.cfssl ];
+      } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \
+           mkdir -p $out; cp *.pem $out'';
+in
+  # make ca derivation sha depend on initca cfssl output
+  pkgs.stdenv.mkDerivation {
+    name = "ca";
+    src = initca';
+    buildCommand = ''
+      mkdir -p $out;
+      cp -r $src/* $out
+    '';
+  }

lib/k8s.nix

@@ -1,6 +1,6 @@
-{ pkgs, masterNode, etcdNodes, clusterHosts, certs, ...}:
+{ pkgs, masterAddress, etcdNodes, clusterHosts, certs, ...}:
 let
-  kubeApiserver = "https://${masterNode}:8443";
+  kubeApiserver = "https://${masterAddress}:8443";
   localApiserver = "http://127.0.0.1:8080";
   etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
   etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
@@ -10,8 +10,8 @@ rec {
     services.etcd = {
       inherit name;
       enable = true;
-      listenClientUrls = ["https://0.0.0.0:2379"];
+      listenClientUrls = [ "https://0.0.0.0:2379" ];
-      listenPeerUrls = ["https://0.0.0.0:2380"];
+      listenPeerUrls = [ "https://0.0.0.0:2380" ];
       peerClientCertAuth = true;
       keyFile  = certs.etcd.key;
       certFile = certs.etcd.cert;
@@ -39,6 +39,7 @@ rec {
 
   kubeNode = instance: {
     services.kubernetes = rec {
+      inherit masterAddress;
       roles = [ "node" ];
       kubeconfig = clientConf instance;
       kubelet = {
@@ -69,8 +70,8 @@ rec {
       roles = [ "master" ];
       kubelet.unschedulable = false;
       apiserver = {
-        bindAddress = "0.0.0.0"; #masterNode;
+        bindAddress = "0.0.0.0"; #masterAddress;
-        advertiseAddress = masterNode;
+        advertiseAddress = masterAddress;
         authorizationMode = [ "Node" "RBAC" ];
         securePort = 8443;
         tlsKeyFile   = certs.apiserver.key;
@@ -80,12 +81,18 @@ rec {
         kubeletClientKeyFile  = certs.apiserver.key;
         kubeletClientCertFile = certs.apiserver.cert;
         serviceAccountKeyFile = certs.apiserver.key;
+        etcd = {
+          servers = etcdEndpoints;
+          keyFile  = certs.apiserver.key;
+          certFile = certs.apiserver.cert;
+          caFile   = certs.ca.cert;
+        };
       };
       scheduler.leaderElect = true;
       controllerManager = {
         leaderElect = true;
         serviceAccountKeyFile = certs.apiserver.key;
-        rootCaFile = certs.ca.cert;
+        # rootCaFile = certs.ca.cert;
         kubeconfig.server = localApiserver;
       };
       scheduler.kubeconfig.server = localApiserver;
@@ -96,7 +103,7 @@ rec {
       	version = "v1.10.0";
         rbac.enable = true;
         rbac.clusterAdmin = true;
-        tokenTtl = 0;
+        # tokenTtl = 0;
      	image = {
           imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
           imageDigest = "sha256:1d2e1229a918f4bc38b5a3f9f5f11302b3e71f8397b492afac7f273a0008776a";
@@ -114,16 +121,9 @@ rec {
 
   kubeConfig = instance: {
     services.kubernetes = {
-      verbose = false;
+      # caFile = certs.ca.cert;
-      caFile = certs.ca.cert;
       flannel.enable = true;
       clusterCidr = "10.10.0.0/16";
-      etcd = {
-        servers = etcdEndpoints;
-        keyFile  = certs.apiserver.key;
-        certFile = certs.apiserver.cert;
-        caFile   = certs.ca.cert;
-      };
       proxy = {
         kubeconfig = clientConf "kube-proxy";
       };
@@ -149,9 +149,9 @@ rec {
     networking = {
       hostName = instance;
       extraHosts = clusterHosts;
-      # nameservers = [ masterNode ];
+      # nameservers = [ masterAddress ];
       # dhcpcd.extraConfig = ''
-      #  static domain_name_servers=${masterNode}
+      #  static domain_name_servers=${masterAddress}
       # '';
       firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
       firewall.allowedTCPPorts = [ 80 443 111 ];
@@ -200,15 +200,5 @@ rec {
         (kubeConfig name)
         (kubeNode name)
       ];
-      services.dockerRegistry = {
-        enable = true;
-        listenAddress = "0.0.0.0";
-        enableDelete = true;
-        enableGarbageCollect = true;
-        extraConfig = {
-          REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}";
-          REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}";
-        };
-      };
     };
 }

lib/pki.nix

@@ -1,4 +1,5 @@
-{ pkgs ? import <nixpkgs> {} }: rec {
+{ pkgs ? import <nixpkgs> {} }:
+let
   ca-config = pkgs.writeText "ca-config.json" ''
   {
       "signing": {
@@ -38,12 +39,20 @@
 
   initca' =
     let
-      ca_csr = gencsr {
+      ca_csr = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
-        name = "kubernetes";
+          key = {
-        cn = "kubernetes";
+              algo = "rsa";
-        o = "kubernetes";
+              size = 2048;
-        hosts = "";
           };
+          names = [
+            {
+              CN = "kubernetes-cluster-ca";
+              O = "NixOS";
+              OU = "services.kubernetes.pki.caSpec";
+              L = "generated";
+            }
+          ];
+        });
     in
       pkgs.runCommand "initca" {
         buildInputs = [ pkgs.cfssl ];
@@ -72,16 +81,43 @@
     mkdir -p $out; cp *.pem $out
   '';
 
-  toSet = cert:
-    {
-      key = "${cert}/cert-key.pem";
-      cert = "${cert}/cert.pem";
-    };
-
   gencert = conf:
+    let crt =
       pkgs.runCommand "${conf.name}" {
         buildInputs = [ pkgs.cfssl ];
       } (cfssl conf);
+    in
+    {
+      key = "${crt}/cert-key.pem";
+      cert = "${crt}/cert.pem";
+    };
+
+  trust = name: hosts:
+    let
+      hosts' = "\"${name}\", " + hosts;
+    in gencert rec {
+      inherit name;
+      csr = gencsr {
+        inherit name;
+        hosts = hosts';
+        cn = name;
+        o = name;
+      };
+    };
+
+  # certToSet = cert:
+  #   {
+  #     key = "${cert}/cert-key.pem";
+  #     cert = "${cert}/cert.pem";
+  #   };
+
+  # builtins.foldl'
+  #   (a: x: a // { ${x} = (certificates.${x}); })
+  #   { inherit ca; }
+  #   (builtins.attrNames certificates)
+in
+{
+    inherit ca;
 
     admin = gencert rec {
         name = "admin";
@@ -112,19 +148,6 @@
       };
     };
 
-  trust = name: hosts:
-    let
-      hosts' = "\"${name}\", " + hosts;
-    in gencert rec {
-      inherit name;
-      csr = gencsr {
-        inherit name;
-        hosts = hosts';
-        cn = name;
-        o = name;
-      };
-    };
-
     kube-proxy = gencert rec {
       name = "kube-proxy";
       csr = gencsr {
@@ -146,3 +169,4 @@
         };
     };
 }
+

nixos/configuration.nix

@@ -20,7 +20,8 @@
 
   services.openssh.enable = true;
   services.nfs.server.enable = true;
-  services.vmwareGuest.enable = true;
+
+  # virtualisation.vmware.guest.enable = true;
 
   programs.fish.enable = true;
   programs.tmux.enable = true;
@@ -29,15 +30,15 @@
 
   disabledModules = [
     # "services/cluster/kubernetes/default.nix"
-    "services/cluster/kubernetes/dns.nix"
+    # "services/cluster/kubernetes/dns.nix"
-    "services/cluster/kubernetes/dashboard.nix"
+    # "services/cluster/kubernetes/dashboard.nix"
   ];
 
   imports = [
     ./users.nix
     ./packages.nix
-    ./overlays/dns.nix
+    # ./overlays/dns.nix
-    ./overlays/dashboard.nix
+    # ./overlays/dashboard.nix
     # ./overlays/kubernetes.nix
   ];
 
@@ -53,7 +54,7 @@
   ];
 
   # The NixOS release to be compatible with for stateful data such as databases.
-  system.stateVersion = "18.09";
+  system.stateVersion = "19.03";
   system.autoUpgrade.enable = false;
 
 }

nixos/overlays/dashboard.nix

@@ -1,330 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-let
-  cfg = config.services.kubernetes.addons.dashboard;
-in {
-  options.services.kubernetes.addons.dashboard = {
-    enable = mkEnableOption "kubernetes dashboard addon";
-
-    rbac = mkOption {
-      description = "Role-based access control (RBAC) options";
-      default = {};
-      type = types.submodule {
-        options = {
-          enable = mkOption {
-            description = "Whether to enable role based access control is enabled for kubernetes dashboard";
-            type = types.bool;
-            default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode;
-          };
-
-          clusterAdmin = mkOption {
-            description = "Whether to assign cluster admin rights to the kubernetes dashboard";
-            type = types.bool;
-            default = false;
-          };
-        };
-      };
-    };
-
-    version = mkOption {
-      description = "Which version of the kubernetes dashboard to deploy";
-      type = types.str;
-      default = "v1.8.3";
-    };
-
-    image = mkOption {
-      description = "Docker image to seed for the kubernetes dashboard container.";
-      type = types.attrs;
-      default = {
-        imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
-        imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0";
-        finalImageTag = cfg.version;
-        sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8";
-      };
-    };
-
-    tokenTtl = mkOption {
-      description = "Expiration time (in seconds) of JWE tokens generated by dashboard. Default: 15 min. 0 - never expires.";
-      type = types.int;
-      default = 15;
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages = [(pkgs.dockerTools.pullImage cfg.image)];
-
-    services.kubernetes.addonManager.addons = {
-      kubernetes-dashboard-deployment = {
-        kind = "Deployment";
-        apiVersion = "apps/v1";
-        metadata = {
-          labels = {
-            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-            k8s-app = "kubernetes-dashboard";
-            version = cfg.version;
-            "kubernetes.io/cluster-service" = "true";
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-          };
-          name = "kubernetes-dashboard";
-          namespace = "kube-system";
-        };
-        spec = {
-          replicas = 1;
-          revisionHistoryLimit = 10;
-          selector.matchLabels."k8s-app" = "kubernetes-dashboard";
-          template = {
-            metadata = {
-              labels = {
-                k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-                k8s-app = "kubernetes-dashboard";
-                version = cfg.version;
-                "kubernetes.io/cluster-service" = "true";
-              };
-              annotations = {
-                "scheduler.alpha.kubernetes.io/critical-pod" = "";
-              };
-            };
-            spec = {
-              priorityClassName = "system-cluster-critical";
-              containers = [{
-                name = "kubernetes-dashboard";
-                image = with cfg.image; "${imageName}:${finalImageTag}";
-                ports = [{
-                  containerPort = 8443;
-                  protocol = "TCP";
-                }];
-                resources = {
-                  limits = {
-                    cpu = "100m";
-                    memory = "300Mi";
-                  };
-                  requests = {
-                    cpu = "100m";
-                    memory = "100Mi";
-                  };
-                };
-                args = [
-                  "--auto-generate-certificates"
-                  "--token-ttl=${toString cfg.tokenTtl}"
-                ];
-                volumeMounts = [{
-                  name = "tmp-volume";
-                  mountPath = "/tmp";
-                } {
-                  name = "kubernetes-dashboard-certs";
-                  mountPath = "/certs";
-                }];
-                livenessProbe = {
-                  httpGet = {
-                    scheme = "HTTPS";
-                    path = "/";
-                    port = 8443;
-                  };
-                  initialDelaySeconds = 30;
-                  timeoutSeconds = 30;
-                };
-              }];
-              volumes = [{
-                name = "kubernetes-dashboard-certs";
-                secret = {
-                  secretName = "kubernetes-dashboard-certs";
-                };
-              } {
-                name = "tmp-volume";
-                emptyDir = {};
-              }];
-              serviceAccountName = "kubernetes-dashboard";
-              tolerations = [{
-                key = "node-role.kubernetes.io/master";
-                effect = "NoSchedule";
-              } {
-                key = "CriticalAddonsOnly";
-                operator = "Exists";
-              }];
-            };
-          };
-        };
-      };
-
-      kubernetes-dashboard-svc = {
-        apiVersion = "v1";
-        kind = "Service";
-        metadata = {
-          labels = {
-            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-            k8s-app = "kubernetes-dashboard";
-            "kubernetes.io/cluster-service" = "true";
-            "kubernetes.io/name" = "KubeDashboard";
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-          };
-          name = "kubernetes-dashboard";
-          namespace  = "kube-system";
-        };
-        spec = {
-          ports = [{
-            port = 443;
-            targetPort = 8443;
-          }];
-          selector.k8s-app = "kubernetes-dashboard";
-        };
-      };
-
-      kubernetes-dashboard-sa = {
-        apiVersion = "v1";
-        kind = "ServiceAccount";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-          };
-          name = "kubernetes-dashboard";
-          namespace = "kube-system";
-        };
-      };
-      kubernetes-dashboard-sec-certs = {
-        apiVersion = "v1";
-        kind = "Secret";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-certs";
-          namespace = "kube-system";
-        };
-        type = "Opaque";
-      };
-      kubernetes-dashboard-sec-kholder = {
-        apiVersion = "v1";
-        kind = "Secret";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-key-holder";
-          namespace = "kube-system";
-        };
-        type = "Opaque";
-      };
-      kubernetes-dashboard-cm = {
-        apiVersion = "v1";
-        kind = "ConfigMap";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-settings";
-          namespace = "kube-system";
-        };
-      };
-    } // (optionalAttrs cfg.rbac.enable
-      (let
-        subjects = [{
-          kind = "ServiceAccount";
-          name = "kubernetes-dashboard";
-          namespace = "kube-system";
-        }];
-        labels = {
-          k8s-app = "kubernetes-dashboard";
-          k8s-addon = "kubernetes-dashboard.addons.k8s.io";
-          "addonmanager.kubernetes.io/mode" = "Reconcile";
-        };
-      in
-        (if cfg.rbac.clusterAdmin then {
-          kubernetes-dashboard-crb = {
-            apiVersion = "rbac.authorization.k8s.io/v1";
-            kind = "ClusterRoleBinding";
-            metadata = {
-              name = "kubernetes-dashboard";
-              inherit labels;
-            };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "ClusterRole";
-              name = "cluster-admin";
-            };
-            inherit subjects;
-          };
-        }
-        else
-        {
-          # Upstream role- and rolebinding as per:
-          # https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/alternative/kubernetes-dashboard.yaml
-          kubernetes-dashboard-role = {
-            apiVersion = "rbac.authorization.k8s.io/v1";
-            kind = "Role";
-            metadata = {
-              name = "kubernetes-dashboard-minimal";
-              namespace = "kube-system";
-              inherit labels;
-            };
-            rules = [
-              # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
-              {
-                apiGroups = [""];
-                resources = ["secrets"];
-                verbs = ["create"];
-              }
-              # Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
-              {
-                apiGroups = [""];
-                resources = ["configmaps"];
-                verbs = ["create"];
-              }
-              # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
-              {
-                apiGroups = [""];
-                resources = ["secrets"];
-                resourceNames = ["kubernetes-dashboard-key-holder"];
-                verbs = ["get" "update" "delete"];
-              }
-              # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
-              {
-                apiGroups = [""];
-                resources = ["configmaps"];
-                resourceNames = ["kubernetes-dashboard-settings"];
-                verbs = ["get" "update"];
-              }
-              # Allow Dashboard to get metrics from heapster.
-              {
-                apiGroups = [""];
-                resources = ["services"];
-                resourceNames = ["heapster"];
-                verbs = ["proxy"];
-              }
-              {
-                apiGroups = [""];
-                resources = ["services/proxy"];
-                resourceNames = ["heapster" "http:heapster:" "https:heapster:"];
-                verbs = ["get"];
-              }
-            ];
-          };
-
-          kubernetes-dashboard-rb = {
-            apiVersion = "rbac.authorization.k8s.io/v1";
-            kind = "RoleBinding";
-            metadata = {
-              name = "kubernetes-dashboard-minimal";
-              namespace = "kube-system";
-              inherit labels;
-            };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "Role";
-              name = "kubernetes-dashboard-minimal";
-            };
-            inherit subjects;
-          };
-        })
-    ));
-  };
-}

nixos/overlays/dns.nix

@@ -1,329 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-
-let
-  version = "1.2.5";
-  cfg = config.services.kubernetes.addons.dns;
-  ports = {
-    dns = 10053;
-    health = 10054;
-    metrics = 10055;
-  };
-in {
-  options.services.kubernetes.addons.dns = {
-    enable = mkEnableOption "kubernetes dns addon";
-
-    clusterIp = mkOption {
-      description = "Dns addon clusterIP";
-
-      # this default is also what kubernetes users
-      default = (
-        concatStringsSep "." (
-          take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
-        ))
-      ) + ".254";
-      type = types.str;
-    };
-
-    clusterDomain = mkOption {
-      description = "Dns cluster domain";
-      default = "cluster.local";
-      type = types.str;
-    };
-
-    replicas = mkOption {
-      description = "Number of DNS pod replicas to deploy in the cluster.";
-      default = 2;
-      type = types.int;
-    };
-
-    reconcileMode = mkOption {
-      description = ''
-        Controls the addon manager reconciliation mode for the DNS addon.
-        See: <link xlink:href="https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md"/>
-      '';
-      default = "Reconcile";
-      type = types.enum [ "Reconcile" "EnsureExists" ];
-    };
-
-    coredns = mkOption {
-      description = "Docker image to seed for the CoreDNS container.";
-      type = types.attrs;
-      default = {
-        imageName = "coredns/coredns";
-        imageDigest = "sha256:33c8da20b887ae12433ec5c40bfddefbbfa233d5ce11fb067122e68af30291d6";
-        finalImageTag = version;
-        sha256 = "13q19rgwapv27xcs664dw502254yw4zw63insf6g2danidv2mg6i";
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.kubernetes.kubelet.seedDockerImages =
-      singleton (pkgs.dockerTools.pullImage cfg.coredns);
-
-    services.kubernetes.addonManager.addons = {
-      coredns-sa = {
-        apiVersion = "v1";
-        kind = "ServiceAccount";
-        metadata = {
-          labels = {
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-            "k8s-app" = "kube-dns";
-            "kubernetes.io/cluster-service" = "true";
-          };
-          name = "coredns";
-          namespace = "kube-system";
-        };
-      };
-
-      coredns-cr = {
-        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-        kind = "ClusterRole";
-        metadata = {
-          labels = {
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-            "k8s-app" = "kube-dns";
-            "kubernetes.io/cluster-service" = "true";
-            "kubernetes.io/bootstrapping" = "rbac-defaults";
-          };
-          name = "system:coredns";
-        };
-        rules = [
-          {
-            apiGroups = [ "" ];
-            resources = [ "endpoints" "services" "pods" "namespaces" ];
-            verbs = [ "list" "watch" ];
-          }
-          {
-            apiGroups = [ "" ];
-            resources = [ "nodes" ];
-            verbs = [ "get" ];
-          }
-        ];
-      };
-
-      coredns-crb = {
-        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-        kind = "ClusterRoleBinding";
-        metadata = {
-          annotations = {
-            "rbac.authorization.kubernetes.io/autoupdate" = "true";
-          };
-          labels = {
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-            "k8s-app" = "kube-dns";
-            "kubernetes.io/cluster-service" = "true";
-            "kubernetes.io/bootstrapping" = "rbac-defaults";
-          };
-          name = "system:coredns";
-        };
-        roleRef = {
-          apiGroup = "rbac.authorization.k8s.io";
-          kind = "ClusterRole";
-          name = "system:coredns";
-        };
-        subjects = [
-          {
-            kind = "ServiceAccount";
-            name = "coredns";
-            namespace = "kube-system";
-          }
-        ];
-      };
-
-      coredns-cm = {
-        apiVersion = "v1";
-        kind = "ConfigMap";
-        metadata = {
-          labels = {
-            "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
-            "k8s-app" = "kube-dns";
-            "kubernetes.io/cluster-service" = "true";
-          };
-          name = "coredns";
-          namespace = "kube-system";
-        };
-        data = {
-          Corefile = ".:${toString ports.dns} {
-            errors
-            health :${toString ports.health}
-            kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
-              pods insecure
-              upstream
-              fallthrough in-addr.arpa ip6.arpa
-            }
-            prometheus :${toString ports.metrics}
-            proxy . /etc/resolv.conf
-            cache 30
-            loop
-            reload
-            loadbalance
-          }";
-        };
-      };
-
-      coredns-deploy = {
-        apiVersion = "extensions/v1beta1";
-        kind = "Deployment";
-        metadata = {
-          labels = {
-            "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
-            "k8s-app" = "kube-dns";
-            "kubernetes.io/cluster-service" = "true";
-            "kubernetes.io/name" = "CoreDNS";
-          };
-          name = "coredns";
-          namespace = "kube-system";
-        };
-        spec = {
-          replicas = cfg.replicas;
-          selector = {
-            matchLabels = { k8s-app = "kube-dns"; };
-          };
-          strategy = {
-            rollingUpdate = { maxUnavailable = 1; };
-            type = "RollingUpdate";
-          };
-          template = {
-            metadata = {
-              labels = {
-                k8s-app = "kube-dns";
-              };
-            };
-            spec = {
-              containers = [
-                {
-                  args = [ "-conf" "/etc/coredns/Corefile" ];
-                  image = with cfg.coredns; "${imageName}:${finalImageTag}";
-                  imagePullPolicy = "Never";
-                  livenessProbe = {
-                    failureThreshold = 5;
-                    httpGet = {
-                      path = "/health";
-                      port = ports.health;
-                      scheme = "HTTP";
-                    };
-                    initialDelaySeconds = 60;
-                    successThreshold = 1;
-                    timeoutSeconds = 5;
-                  };
-                  name = "coredns";
-                  ports = [
-                    {
-                      containerPort = ports.dns;
-                      name = "dns";
-                      protocol = "UDP";
-                    }
-                    {
-                      containerPort = ports.dns;
-                      name = "dns-tcp";
-                      protocol = "TCP";
-                    }
-                    {
-                      containerPort = ports.metrics;
-                      name = "metrics";
-                      protocol = "TCP";
-                    }
-                  ];
-                  resources = {
-                    limits = {
-                      memory = "170Mi";
-                    };
-                    requests = {
-                      cpu = "100m";
-                      memory = "70Mi";
-                    };
-                  };
-                  securityContext = {
-                    allowPrivilegeEscalation = false;
-                    capabilities = {
-                      drop = [ "all" ];
-                    };
-                    readOnlyRootFilesystem = true;
-                  };
-                  volumeMounts = [
-                    {
-                      mountPath = "/etc/coredns";
-                      name = "config-volume";
-                      readOnly = true;
-                    }
-                  ];
-                }
-              ];
-              dnsPolicy = "Default";
-              nodeSelector = {
-                "beta.kubernetes.io/os" = "linux";
-              };
-              serviceAccountName = "coredns";
-              tolerations = [
-                {
-                  effect = "NoSchedule";
-                  key = "node-role.kubernetes.io/master";
-                }
-                {
-                  key = "CriticalAddonsOnly";
-                  operator = "Exists";
-                }
-              ];
-              volumes = [
-                {
-                  configMap = {
-                    items = [
-                      {
-                        key = "Corefile";
-                        path = "Corefile";
-                      }
-                    ];
-                    name = "coredns";
-                  };
-                  name = "config-volume";
-                }
-              ];
-            };
-          };
-        };
-      };
-
-      coredns-svc = {
-        apiVersion = "v1";
-        kind = "Service";
-        metadata = {
-          annotations = {
-            "prometheus.io/port" = toString ports.metrics;
-            "prometheus.io/scrape" = "true";
-          };
-          labels = {
-            "addonmanager.kubernetes.io/mode" = "Reconcile";
-            "k8s-app" = "kube-dns";
-            "kubernetes.io/cluster-service" = "true";
-            "kubernetes.io/name" = "CoreDNS";
-          };
-          name = "kube-dns";
-          namespace = "kube-system";
-        };
-        spec = {
-          clusterIP = cfg.clusterIp;
-          ports = [
-            {
-              name = "dns";
-              port = 53;
-              targetPort = ports.dns;
-              protocol = "UDP";
-            }
-            {
-              name = "dns-tcp";
-              port = 53;
-              targetPort = ports.dns;
-              protocol = "TCP";
-            }
-          ];
-          selector = { k8s-app = "kube-dns"; };
-        };
-      };
-    };
-
-    services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
-  };
-}

nixos/overlays/overlays.nix

@@ -1,6 +0,0 @@
-self: super: {
-  super.config.services.kubernetes.addons.dns =
-    super.callPackage ./dns.nix {};
-  super.config.services.kubernetes.addons.dashboard =
-    super.callPackage ./dashboard.nix {};
-}

nixos/packages.nix

@@ -33,6 +33,7 @@
       ebtables
       ethtool
       socat
+      docker
     ];
   in
   k8s ++