Major revamp.

2018-04-17 21:35:56 +02:00
parent 72285487d6
commit 0ea0abd316
32 changed files with 2600 additions and 353 deletions
--- a/base/pki.nix
+++ b/base/pki.nix
@@ -1,124 +0,0 @@
-with import <nixpkgs> {};
-let
-  ca-config = pkgs.writeText "ca-config.json" ''
-  {
-      "signing": {
-          "default": {
-              "expiry": "43800h"
-          },
-          "profiles": {
-              "server": {
-                  "expiry": "43800h",
-                  "usages": [
-                      "signing",
-                      "key encipherment",
-                      "server auth"
-                  ]
-              },
-              "client": {
-                  "expiry": "43800h",
-                  "usages": [
-                      "signing",
-                      "key encipherment",
-                      "client auth"
-                  ]
-              }
-          }
-      }
-  }
-  '';
-
-  csr = args: pkgs.writeText "${args.cn}-cert.json" ''
-    {
-    "CN": "${args.cn}",
-    "hosts": [ ${args.hosts} ],
-    "key": {
-        "algo": "rsa",
-        "size": 2048
-    },
-    "names": [
-        {
-            "C": "NO",
-            "L": "Tromsø",
-            "O": "Serit IT Partner Tromsø AS",
-            "OU": "",
-            "ST": ""
-        }
-    ]
-    }
-  '';
-
-  ca-csr = csr  { cn = "kubernetes"; hosts = ""; };
-  ca = pkgs.runCommand "ca-cert" {
-      buildInputs = [ pkgs.cfssl ];
-    } '' cfssl genkey -initca ${ca-csr} | cfssljson -bare ca; \
-         mkdir -p $out; cp *.pem $out'';
-
-  ca_cert = "${ca}/ca.pem";
-  ca_key = "${ca}/ca-key.pem";
-
-  cfssl = name: profile: ''
-    cfssl gencert -ca ${ca_cert} -ca-key ${ca_key} \
-    -config=${ca-config} -profile=${profile} ${name} | cfssljson -bare cert; \
-    mkdir -p $out; cp *.pem $out
-  '';
-in
-  rec {
-    inherit ca_cert;
-    inherit ca_key;
-    inherit csr;
-
-    mkCert = cert:
-    pkgs.runCommand "${cert.name}-cert" {
-      buildInputs = [ pkgs.cfssl ];
-    } (cfssl cert.csr cert.profile);
-
-  # server-cert = mkCert {
-  #     name = "kubernetes";
-  #     csr = csr {
-  #       cn = "kubernetes";
-  #       hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
-  #     };
-  #     profile = "server";
-  # };
-
-  # etcd0-cert = mkCert {
-  #   name = "etcd0";
-  #   csr = csr {
-  #     cn = "etcd0";
-  #     hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
-  #   };
-  #   profile = "peer";
-  # };
-
-  # etcd1-cert = mkCert {
-  #   name = "etcd1";
-  #   csr = csr {
-  #     cn = "etcd1";
-  #     hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
-  #   };
-  #   profile = "peer";
-  # };
-
-  # client-cert = mkCert {
-  #   name = "client";
-  #   csr = csr {
-  #     cn = "client";
-  #     hosts = '''';
-  #   };
-  #   profile = "client";
-  # };
-
-  # server_key = "${server-cert}/cert-key.pem";
-  # server_cert = "${server-cert}/cert.pem";
-
-  # etcd0_key = "${etcd0-cert}/cert-key.pem";
-  # etcd0_cert = "${etcd0-cert}/cert.pem";
-
-  # etcd1_key = "${etcd1-cert}/cert-key.pem";
-  # etcd1_cert = "${etcd1-cert}/cert.pem";
-
-  # client_key = "${client-cert}/cert-key.pem";
-  # client_cert = "${client-cert}/cert.pem";
-
-  }
--- a/bin/teardown.sh
+++ b/bin/teardown.sh
@@ -17,6 +17,6 @@ nixops modify -d $d $f
 nixops deploy -d $d
 rm $f

+nixops reboot -d $d
 nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd"
-nixops ssh-for-each -d $d reboot

--- a/fs0.nix
+++ b/fs0.nix
@@ -0,0 +1,63 @@
+with import <nixpkgs> {};
+let
+  pki = pkgs.callPackage ./lib/pki.nix {};
+  certs = {
+    ca = pki.ca;
+    fs = pki.etcd ''
+        "fs0-0",
+        "fs0-1",
+        "10.253.18.106",
+        "10.1.2.164",
+        "127.0.0.1"
+      '';
+  };
+  clusterHosts = ''
+    10.253.18.106 fs0-0
+    10.1.2.164    fs0-1
+  '';
+
+  nixosConfig = node: {
+    imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ];
+    networking = {
+      hostName = node;
+      extraHosts = clusterHosts;
+      # firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      # firewall.allowedTCPPorts = [ 80 443 ];
+    };
+    environment.systemPackages = [ pkgs.tshark ];
+    services.glusterfs = {
+      enable = true;
+      tlsSettings = {
+        caCert = certs.ca.cert;
+        tlsKeyPath = certs.fs.key;
+        tlsPem = certs.fs.cert;
+      };
+    };
+  };
+in
+{
+  fs0-0 = { ... }:
+    let
+      base = nixosConfig "fs0-0";
+    in
+    {
+      deployment.targetHost = "10.253.18.106";
+      require = [ base ];
+      services.nfs.server = {
+        enable=true;
+        exports= ''
+          /data/vol1 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+        '';
+      };
+      networking.firewall.allowedTCPPorts = [ 111 2049 ];
+      networking.firewall.allowedUDPPorts = [ 111 2049 ];
+    };
+  fs0-1 = { ... }:
+    let
+      base = nixosConfig "fs0-1";
+    in
+    {
+      deployment.targetHost = "10.1.2.164";
+      require = [ base ];
+    };
+}
--- a/git.nix
+++ b/git.nix
@@ -1,17 +0,0 @@
-{
-  git01 = { config, lib, pkgs, ... }:
-  {
-    deployment.targetHost = "10.253.18.103";
-    networking.hostName = "git01"; # Define your hostname
-    imports = [ ./hw/git01.nix ./base/configuration.nix ];
-    services.nfs.server = {
-      enable=true;
-      exports= ''
-        /data 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-        /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-      '';
-    };
-    networking.firewall.allowedTCPPorts = [2049 111 20048];
-    networking.firewall.allowedUDPPorts = [2049 111 20048];
-  };
-}
--- a/hardware-configuration/fs0-0.nix
+++ b/hardware-configuration/fs0-0.nix
@@ -0,0 +1,29 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/6b3d4c49-9719-49b3-8210-d53374cd0eff";
+      fsType = "ext4";
+    };
+  fileSystems."/var/log" =
+    { device = "/dev/disk/by-uuid/c1e78683-4fde-4029-a9f3-7631df649b2f";
+      fsType = "ext4";
+    };
+  fileSystems."/data" =
+    { device = "/dev/sdb1";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
--- a/hardware-configuration/fs0-1.nix
+++ b/hardware-configuration/fs0-1.nix
@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/e8820516-9c21-46f4-9dde-a7a77bf67bbd";
+      fsType = "ext4";
+    };
+  fileSystems."/var/log" =
+    { device = "/dev/disk/by-uuid/c130b88c-0699-4836-b967-47bdee0a5453";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
+
--- a/hardware-configuration/k0-0.nix
+++ b/hardware-configuration/k0-0.nix
--- a/hardware-configuration/k0-1.nix
+++ b/hardware-configuration/k0-1.nix
--- a/hardware-configuration/k0-2.nix
+++ b/hardware-configuration/k0-2.nix
--- a/hardware-configuration/k0-3.nix
+++ b/hardware-configuration/k0-3.nix
--- a/hardware-configuration/k0-4.nix
+++ b/hardware-configuration/k0-4.nix
@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/241cdf57-a2b5-482d-a522-01725badc859";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
--- a/hardware-configuration/k0-5.nix
+++ b/hardware-configuration/k0-5.nix
@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/67441b95-19f2-484d-b57b-3f4b2a55f3cc";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
--- a/hardware-configuration/k1-0.nix
+++ b/hardware-configuration/k1-0.nix
@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/2e7ba83d-014f-4ef5-a1ce-fc9e34ce7b83";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
--- a/hardware-configuration/k1-1.nix
+++ b/hardware-configuration/k1-1.nix
@@ -0,0 +1,21 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/70b9d730-9cb6-48e2-8e00-fa78c8feefdf";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 1;
+}
--- a/k0.nix
+++ b/k0.nix
@@ -0,0 +1,74 @@
+with import <nixpkgs> {};
+let
+  pki = pkgs.callPackage ./lib/pki.nix {};
+  certs = {
+    ca = pki.ca;
+    apiserver = pki.apiserver ''
+          "10.253.18.100",
+          "10.0.0.1",
+          "127.0.0.1",
+          "kubernetes",
+          "etcd0",
+          "k0-0"
+        '';
+    kube-proxy = pki.kube-proxy;
+    admin = pki.admin;
+    etcd = pki.etcd ''
+        "etcd0",
+        "etcd1",
+        "etcd2",
+        "10.253.18.100",
+        "10.253.18.101",
+        "10.253.18.102",
+        "127.0.0.1"
+      '';
+      k0-0 = pki.worker { name = "k0-0"; ip = "10.253.18.100"; };
+      k0-1 = pki.worker { name = "k0-1"; ip = "10.253.18.101"; };
+      k0-2 = pki.worker { name = "k0-2"; ip = "10.253.18.102"; };
+      k0-3 = pki.worker { name = "k0-3"; ip = "10.253.18.103"; };
+      k0-4 = pki.worker { name = "k0-4"; ip = "10.253.18.107"; };
+      k0-5 = pki.worker { name = "k0-5"; ip = "10.253.18.108"; };
+  };
+  cluster = callPackage ./lib/k8s.nix {
+    kubeMaster = "10.253.18.100";
+    etcdNodes = [ "etcd0" "etcd1" "etcd2" ];
+    clusterHosts = ''
+          10.253.18.100 k0-0 etcd0 kubernetes
+          10.253.18.101 k0-1 etcd1
+          10.253.18.102 k0-2 etcd2
+          10.253.18.103 k0-3
+          10.253.18.107 k0-4
+          10.253.18.108 k0-5
+          10.253.18.106 fs0-0
+          10.1.2.164    fs0-1
+    '';
+    inherit certs;
+  };
+in
+{
+  k0-0 = cluster.apiserver "k0-0" "10.253.18.100" "etcd0";
+  k0-1 = cluster.server    "k0-1" "10.253.18.101" "etcd1";
+  k0-2 = cluster.server    "k0-2" "10.253.18.102" "etcd2";
+  k0-3 = cluster.worker    "k0-3" "10.253.18.103";
+  k0-4 = cluster.worker    "k0-4" "10.253.18.107";
+  k0-5 = cluster.worker    "k0-5" "10.253.18.108";
+
+  # k0-3 = { config, lib, pkgs, ... }:
+  #   let
+  #     instance = "k0-3";
+  #     base = k8s.nixosConfig instance;
+  #   in
+  #   {
+  #     deployment.targetHost = "10.253.18.103";
+  #     require = [ k8s.base (k8s.kubeConfig instance) (k8s.kubeNode instance) ];
+  #     services.kubernetes.addons.dns.enable = false;
+  #     services.nfs.server = {
+  #       enable=true;
+  #       exports= ''
+  #         /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+  #       '';
+  #     };
+  #     networking.firewall.allowedTCPPorts = [ 111 2049 ];
+  #     networking.firewall.allowedUDPPorts = [ 111 2049 ];
+  #   };
+}
--- a/k8s.nix
+++ b/k8s.nix
@@ -1,209 +0,0 @@
-with import ./certs.nix;
-let
-  pkgs = import <nixpkgs> {};
-
-  kube_apiserver = "https://10.253.18.100:443";
-  etcdServers = [ "etcd0" "etcd1" "etcd2" ];
-  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
-  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdServers;
-
-  etcdConfig = name: {
-    services.etcd = {
-      inherit name;
-      enable = true;
-      listenClientUrls = ["https://0.0.0.0:2379"];
-      listenPeerUrls = ["https://0.0.0.0:2380"];
-      peerClientCertAuth = true;
-      keyFile  = etcd_key;
-      certFile = etcd_cert;
-      trustedCaFile = ca_pem;
-      advertiseClientUrls = [ "https://${name}:2379" ];
-      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
-      initialCluster = etcdCluster;
-    };
-    environment.variables = {
-      ETCDCTL_KEY_FILE  = "${etcd_client_key}";
-      ETCDCTL_CERT_FILE = "${etcd_client_cert}";
-      ETCDCTL_CA_FILE   = "${ca_pem}";
-      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
-    };
-    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
-    systemd.services.flannel.after = [ "etcd.service" ];
-  };
-
-   kubeconfig = {
-    caFile = ca_pem;
-    keyFile = worker_key;
-    certFile = worker_cert;
-    server = kube_apiserver;
-  };
-
-  kubeNode = {
-    services.kubernetes = {
-      roles = [ "node" ];
-      kubeconfig = {
-        server   = kube_apiserver;
-        keyFile  = worker_key;
-        certFile = worker_cert;
-        caFile   = ca_pem;
-      };
-      kubelet = {
-        enable = true;
-        clientCaFile  = ca_pem;
-        tlsKeyFile    = worker_key;
-        tlsCertFile   = worker_cert;
-        networkPlugin = null;
-        # clusterDns    = "10.253.18.100";
-        clusterDns    = "10.0.0.254";
-        inherit kubeconfig;
-      };
-    };
-    networking = {
-      firewall = {
-        enable = true;
-        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
-        allowedTCPPorts = [ 53 4194 10250 ];
-        allowedUDPPorts = [ 53 ];
-        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
-      };
-    };
-    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
-  };
-
-  kubeMaster = {
-    services.kubernetes = {
-      roles = [ "master" ];
-      kubelet.unschedulable = true;
-      apiserver = {
-        address = "0.0.0.0";
-        publicAddress = "0.0.0.0";
-        advertiseAddress = "10.253.18.100";
-        securePort = 443;
-        tlsKeyFile   = apiserver_key;
-        tlsCertFile  = apiserver_cert;
-        clientCaFile = ca_pem;
-        kubeletClientCaFile   = ca_pem;
-        kubeletClientKeyFile  = worker_key;
-        kubeletClientCertFile = worker_cert;
-        serviceAccountKeyFile = apiserver_key;
-      };
-      scheduler.leaderElect = true;
-      controllerManager = {
-        leaderElect = true;
-        serviceAccountKeyFile = apiserver_key;
-        rootCaFile = ca_pem;
-        inherit kubeconfig;
-      };
-      addons.dashboard.enable = true;
-      addons.dns.enable = true;
-    };
-    networking.firewall = {
-      allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
-      # allowedUDPPorts = [ 4053 ];
-    };
-    environment.systemPackages = [ pkgs.kubernetes-helm ];
-  };
-
-  kubeConfig = {
-    services.kubernetes = {
-      verbose = false;
-      caFile = ca_pem;
-      flannel.enable = true;
-      clusterCidr = "10.10.0.0/16";
-      etcd = {
-        servers = etcdEndpoints;
-        keyFile  = etcd_client_key;
-        certFile = etcd_client_cert;
-        caFile   = ca_pem;
-      };
-      proxy = {
-        inherit kubeconfig;
-      };
-    };
-  };
-
-  baseConfig = node: {
-    imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
-    networking = {
-      hostName = node;
-      extraHosts = ''
-        10.253.18.100 etcd0 kubernetes
-        10.253.18.101 etcd1
-        10.253.18.102 etcd2
-      '';
-      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
-      firewall.allowedTCPPorts = [ 80 443 ];
-    };
-    environment.systemPackages = [ pkgs.tshark ];
-  };
-
-  minion = host: ip: { config, lib, pkgs, ... }:
-    let
-      inherit host;
-      base = baseConfig host;
-    in
-    {
-      deployment.targetHost = ip;
-      require = [ base kubeConfig kubeNode ];
-      services.kubernetes.addons.dns.enable = false;
-    };
-in
-{
-  k8s0-0 = { config, lib, pkgs, ... }:
-    let
-      base = baseConfig "k8s0-0";
-      etcd = etcdConfig "etcd0";
-    in
-    {
-      deployment.targetHost = "10.253.18.100";
-      require = [ base etcd kubeConfig kubeMaster kubeNode ];
-      services.dockerRegistry = {
-        enable = true;
-        listenAddress = "0.0.0.0";
-        extraConfig = {
-          REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver_cert}";
-          REGISTRY_HTTP_TLS_KEY = "${apiserver_key}";
-        };
-      };
-    };
-
-  k8s0-1 = { config, lib, pkgs, ... }:
-    let
-      base = baseConfig "k8s0-1";
-      etcd = etcdConfig "etcd1";
-    in
-    {
-      deployment.targetHost = "10.253.18.101";
-      require = [ base etcd kubeConfig kubeNode ];
-      services.kubernetes.addons.dns.enable = false;
-    };
-
-  k8s0-2 = { config, lib, pkgs, ... }:
-    let
-      base = baseConfig "k8s0-2";
-      etcd = etcdConfig "etcd2";
-    in
-    {
-      deployment.targetHost = "10.253.18.102";
-      require = [ base etcd kubeConfig kubeNode ];
-      services.kubernetes.addons.dns.enable = false;
-    };
-
-  k8s0-3 = { config, lib, pkgs, ... }:
-    let
-      base = baseConfig "k8s0-3";
-    in
-    {
-      deployment.targetHost = "10.253.18.103";
-      require = [ base kubeConfig kubeNode ];
-      services.kubernetes.addons.dns.enable = false;
-      services.nfs.server = {
-        enable=true;
-        exports= ''
-          /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-        '';
-      };
-      networking.firewall.allowedTCPPorts = [ 111 2049 ];
-      networking.firewall.allowedUDPPorts = [ 111 2049 ];
-    };
-}
--- a/lib/certs.nix
+++ b/lib/certs.nix
--- a/lib/k8s.nix
+++ b/lib/k8s.nix
@@ -0,0 +1,173 @@
+{ pkgs, kubeMaster, etcdNodes, clusterHosts, certs, ...}:
+let
+  kubeApiserver = "https://${kubeMaster}:443";
+  localApiserver = "https://127.0.0.1:8080";
+  etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
+  etcdCluster =  builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
+in
+rec {
+  etcdConfig = name: {
+    services.etcd = {
+      inherit name;
+      enable = true;
+      listenClientUrls = ["https://0.0.0.0:2379"];
+      listenPeerUrls = ["https://0.0.0.0:2380"];
+      peerClientCertAuth = true;
+      keyFile  = certs.etcd.key;
+      certFile = certs.etcd.cert;
+      trustedCaFile = certs.ca.cert;
+      advertiseClientUrls = [ "https://${name}:2379" ];
+      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
+      initialCluster = etcdCluster;
+    };
+    environment.variables = {
+      ETCDCTL_KEY_FILE  = "${certs.admin.key}";
+      ETCDCTL_CERT_FILE = "${certs.admin.cert}";
+      ETCDCTL_CA_FILE   = "${certs.ca.cert}";
+      ETCDCTL_PEERS     = "https://127.0.0.1:2379";
+    };
+    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+    systemd.services.flannel.after = [ "etcd.service" ];
+  };
+
+   clientConf = instance: {
+    server = kubeApiserver;
+    keyFile = certs.${instance}.key;
+    certFile = certs.${instance}.cert;
+    caFile = certs.ca.cert;
+  };
+
+  kubeNode = instance: {
+    services.kubernetes = rec {
+      roles = [ "node" ];
+      kubeconfig = clientConf instance;
+      kubelet = {
+        enable = true;
+        clientCaFile  = certs.ca.cert;
+        tlsKeyFile    = certs.${instance}.key;
+        tlsCertFile   = certs.${instance}.cert;
+        networkPlugin = null;
+        clusterDns    = "10.0.0.254";
+        extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice";
+        inherit kubeconfig;
+      };
+    };
+    networking = {
+      firewall = {
+        enable = true;
+        # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
+        allowedTCPPorts = [ 53 4194 10250 ];
+        allowedUDPPorts = [ 53 ];
+        extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
+      };
+    };
+    virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
+    # systemd.services.kube-proxy.path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod];
+  };
+
+  kubeMaster = {
+    services.kubernetes = {
+      roles = [ "master" ];
+      kubelet.unschedulable = true;
+      apiserver = {
+        address = kubeMaster;
+        advertiseAddress = kubeMaster;
+        authorizationMode = [ "Node" "RBAC" ];
+        securePort = 443;
+        tlsKeyFile   = certs.apiserver.key;
+        tlsCertFile  = certs.apiserver.cert;
+        clientCaFile = certs.ca.cert;
+        kubeletClientCaFile   = certs.ca.cert;
+        kubeletClientKeyFile  = certs.apiserver.key;
+        kubeletClientCertFile = certs.apiserver.cert;
+        serviceAccountKeyFile = certs.apiserver.key;
+      };
+      scheduler.leaderElect = true;
+      controllerManager = {
+        leaderElect = true;
+        serviceAccountKeyFile = certs.apiserver.key;
+        rootCaFile = certs.ca.cert;
+        kubeconfig.server = localApiserver;
+      };
+      scheduler.kubeconfig.server = localApiserver;
+      addons.dashboard.enable = true;
+      addons.dns.enable = true;
+    };
+    networking.firewall = {
+      allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
+      # allowedUDPPorts = [ 4053 ];
+    };
+    environment.systemPackages = [ pkgs.kubernetes-helm ];
+  };
+
+  kubeConfig = instance: {
+    services.kubernetes = {
+      verbose = false;
+      caFile = certs.ca.cert;
+      flannel.enable = true;
+      clusterCidr = "10.10.0.0/16";
+      etcd = {
+        servers = etcdEndpoints;
+        keyFile  = certs.apiserver.key;
+        certFile = certs.apiserver.cert;
+        caFile   = certs.ca.cert;
+      };
+      proxy = {
+        kubeconfig = clientConf "kube-proxy";
+      };
+    };
+  };
+
+  nixosConfig = node: {
+    imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ];
+    networking = {
+      hostName = node;
+      extraHosts = clusterHosts;
+      firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
+      firewall.allowedTCPPorts = [ 80 443 ];
+    };
+    environment.systemPackages = [ pkgs.tshark ];
+  };
+
+  worker = host: ip: { config, lib, pkgs, ... }:
+    let
+      instance = host;
+      base = nixosConfig host;
+    in
+    {
+      deployment.targetHost = ip;
+      require = [ base (kubeConfig instance) (kubeNode instance) ];
+      services.kubernetes.addons.dns.enable = false;
+    };
+
+  server = host: etc: ip: { config, lib, pkgs, ... }:
+    let
+      instance = host;
+      base = nixosConfig instance;
+      etcd = etcdConfig etc;
+    in
+    {
+      deployment.targetHost = ip;
+      require = [ base etcd (kubeConfig instance) (kubeNode instance) ];
+      services.kubernetes.addons.dns.enable = false;
+    };
+
+  apiserver = host: ip: etc: { config, lib, pkgs, ... }:
+    let
+      instance = host;
+      base = nixosConfig instance;
+      etcd = etcdConfig etc;
+    in
+    {
+      deployment.targetHost = ip;
+      require = [ base etcd (kubeConfig instance) kubeMaster (kubeNode instance) ];
+      services.dockerRegistry = {
+        enable = true;
+        listenAddress = "0.0.0.0";
+        extraConfig = {
+          REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}";
+          REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}";
+        };
+      };
+    };
+}
--- a/lib/pki.nix
+++ b/lib/pki.nix
@@ -0,0 +1,124 @@
+{ pkgs ? import <nixpkgs> {} }: rec {
+  ca-config = pkgs.writeText "ca-config.json" ''
+  {
+      "signing": {
+          "default": {
+              "expiry": "8760h"
+          },
+          "profiles": {
+              "kubernetes": {
+                  "usages": [
+                      "signing",
+                      "key encipherment",
+                      "server auth",
+                      "client auth"
+                  ],
+                  "expiry": "8760h"
+              }
+          }
+      }
+  }
+  '';
+
+  gencsr = args: pkgs.writeText "${args.name}-csr.json" ''
+    {
+      "CN": "${args.cn}",
+      "hosts": [ ${args.hosts} ],
+      "key": {
+          "algo": "rsa",
+          "size": 2048
+      },
+      "names": [
+          {
+            "O": "${args.o}"
+          }
+      ]
+    }
+  '';
+
+  initca =
+    let
+      ca_csr = gencsr {
+        name = "kubernetes";
+        cn = "kubernetes";
+        o = "kubernetes";
+        hosts = "";
+      };
+    in
+      pkgs.runCommand "ca" {
+        buildInputs = [ pkgs.cfssl ];
+      } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \
+           mkdir -p $out; cp *.pem $out'';
+  ca = {
+    key = "${initca}/ca-key.pem";
+    cert = "${initca}/ca.pem";
+  };
+
+  cfssl = conf: ''
+    cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
+    -config=${ca-config} -profile=kubernetes ${conf.csr} | \
+    cfssljson -bare cert; \
+    mkdir -p $out; cp *.pem $out
+  '';
+
+  gencert = conf:
+    let
+      drv = pkgs.runCommand "${conf.name}" {
+        buildInputs = [ pkgs.cfssl ];
+      } (cfssl conf);
+    in
+      {
+        key = "${drv}/cert-key.pem";
+        cert = "${drv}/cert.pem";
+      };
+
+  admin = gencert rec {
+      name = "admin";
+      csr = gencsr {
+        inherit name;
+        cn = "admin";
+        o = "system:masters";
+        hosts = "";
+      };
+  };
+
+  apiserver = hosts:
+    gencert rec {
+      name = "kubernetes";
+      csr = gencsr {
+        inherit name hosts;
+        cn = "kubernetes";
+        o = "kubernetes";
+      };
+  };
+
+  etcd = hosts: gencert rec {
+    name = "etcd";
+    csr = gencsr {
+      inherit name hosts;
+      cn = "etcd";
+      o = "kubernetes";
+    };
+  };
+
+  kube-proxy = gencert rec {
+    name = "kube-proxy";
+    csr = gencsr {
+      inherit name;
+      cn = "system:kube-proxy";
+      o = "system:node-proxier";
+      hosts = "";
+    };
+  };
+
+  worker = instance:
+    gencert rec {
+      name = instance.name;
+      csr = gencsr {
+        inherit name;
+        cn = "system:node:${instance.name}";
+        o = "system:nodes";
+        hosts = ''"${instance.name}","${instance.ip}"'';
+      };
+  };
+}
--- a/nixos/configuration.nix
+++ b/nixos/configuration.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, ... }:
+{ config, pkgs, lib, ... }:
 {
  # Use the GRUB 2 boot loader.
  boot.loader.grub.enable = true;
@@ -29,5 +29,16 @@

  security.rtkit.enable = true;

-  imports = [ ./users.nix ./packages.nix ];
+  disabledModules = [ "services/cluster/kubernetes/default.nix" ];
+
+  imports = [
+    ./users.nix
+    ./packages.nix
+    ./overlays/kubernetes.nix
+  ];
+
+  nixpkgs.overlays = [
+    (import ./overlays/overlays.nix)
+  ];
+
 }
--- a/nixos/desktop.nix
+++ b/nixos/desktop.nix
--- a/nixos/nix-home.nix
+++ b/nixos/nix-home.nix
--- a/nixos/overlays/default.nix
+++ b/nixos/overlays/default.nix
@@ -0,0 +1,593 @@
+{
+  callPackage,
+  coreutils,
+  docker,
+  e2fsprogs,
+  findutils,
+  go,
+  jshon,
+  jq,
+  lib,
+  pkgs,
+  pigz,
+  nixUnstable,
+  perl,
+  runCommand,
+  rsync,
+  shadow,
+  stdenv,
+  storeDir ? builtins.storeDir,
+  utillinux,
+  vmTools,
+  writeReferencesToFile,
+  writeScript,
+  writeText,
+}:
+
+# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
+
+rec {
+
+  examples = import ./examples.nix {
+    inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb;
+  };
+
+  pullImage = callPackage ./pull.nix {};
+
+  # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
+  # And we cannot untar it, because then we cannot preserve permissions ecc.
+  tarsum = runCommand "tarsum" {
+    buildInputs = [ go ];
+  } ''
+    mkdir tarsum
+    cd tarsum
+
+    cp ${./tarsum.go} tarsum.go
+    export GOPATH=$(pwd)
+    mkdir -p src/github.com/docker/docker/pkg
+    ln -sT ${docker.src}/components/engine/pkg/tarsum src/github.com/docker/docker/pkg/tarsum
+    go build
+
+    cp tarsum $out
+  '';
+
+  # buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
+  mergeDrvs = {
+    derivations,
+    onlyDeps ? false
+  }:
+    runCommand "merge-drvs" {
+      inherit derivations onlyDeps;
+    } ''
+      if [[ -n "$onlyDeps" ]]; then
+        echo $derivations > $out
+        exit 0
+      fi
+
+      mkdir $out
+      for derivation in $derivations; do
+        echo "Merging $derivation..."
+        if [[ -d "$derivation" ]]; then
+          # If it's a directory, copy all of its contents into $out.
+          cp -drf --preserve=mode -f $derivation/* $out/
+        else
+          # Otherwise treat the derivation as a tarball and extract it
+          # into $out.
+          tar -C $out -xpf $drv || true
+        fi
+      done
+    '';
+
+  # Helper for setting up the base files for managing users and
+  # groups, only if such files don't exist already. It is suitable for
+  # being used in a runAsRoot script.
+  shadowSetup = ''
+    export PATH=${shadow}/bin:$PATH
+    mkdir -p /etc/pam.d
+    if [[ ! -f /etc/passwd ]]; then
+      echo "root:x:0:0::/root:${stdenv.shell}" > /etc/passwd
+      echo "root:!x:::::::" > /etc/shadow
+    fi
+    if [[ ! -f /etc/group ]]; then
+      echo "root:x:0:" > /etc/group
+      echo "root:x::" > /etc/gshadow
+    fi
+    if [[ ! -f /etc/pam.d/other ]]; then
+      cat > /etc/pam.d/other <<EOF
+    account sufficient pam_unix.so
+    auth sufficient pam_rootok.so
+    password requisite pam_unix.so nullok sha512
+    session required pam_unix.so
+    EOF
+    fi
+    if [[ ! -f /etc/login.defs ]]; then
+      touch /etc/login.defs
+    fi
+  '';
+
+  # Run commands in a virtual machine.
+  runWithOverlay = {
+    name,
+    fromImage ? null,
+    fromImageName ? null,
+    fromImageTag ? null,
+    diskSize ? 1024,
+    preMount ? "",
+    postMount ? "",
+    postUmount ? ""
+  }:
+    vmTools.runInLinuxVM (
+      runCommand name {
+        preVM = vmTools.createEmptyImage {
+          size = diskSize;
+          fullName = "docker-run-disk";
+        };
+        inherit fromImage fromImageName fromImageTag;
+
+        buildInputs = [ utillinux e2fsprogs jshon rsync ];
+      } ''
+      rm -rf $out
+
+      mkdir disk
+      mkfs /dev/${vmTools.hd}
+      mount /dev/${vmTools.hd} disk
+      cd disk
+
+      if [[ -n "$fromImage" ]]; then
+        echo "Unpacking base image..."
+        mkdir image
+        tar -C image -xpf "$fromImage"
+
+        # If the image name isn't set, read it from the image repository json.
+        if [[ -z "$fromImageName" ]]; then
+          fromImageName=$(jshon -k < image/repositories | head -n 1)
+          echo "From-image name wasn't set. Read $fromImageName."
+        fi
+
+        # If the tag isn't set, use the name as an index into the json
+        # and read the first key found.
+        if [[ -z "$fromImageTag" ]]; then
+          fromImageTag=$(jshon -e $fromImageName -k < image/repositories \
+                         | head -n1)
+          echo "From-image tag wasn't set. Read $fromImageTag."
+        fi
+
+        # Use the name and tag to get the parent ID field.
+        parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
+                   < image/repositories)
+      fi
+
+      # Unpack all of the parent layers into the image.
+      lowerdir=""
+      while [[ -n "$parentID" ]]; do
+        echo "Unpacking layer $parentID"
+        mkdir -p image/$parentID/layer
+        tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
+        rm image/$parentID/layer.tar
+
+        find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
+
+        # Get the next lower directory and continue the loop.
+        lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
+        parentID=$(cat image/$parentID/json \
+                  | (jshon -e parent -u 2>/dev/null || true))
+      done
+
+      mkdir work
+      mkdir layer
+      mkdir mnt
+
+      ${lib.optionalString (preMount != "") ''
+        # Execute pre-mount steps
+        echo "Executing pre-mount steps..."
+        ${preMount}
+      ''}
+
+      if [ -n "$lowerdir" ]; then
+        mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
+      else
+        mount --bind layer mnt
+      fi
+
+      ${lib.optionalString (postMount != "") ''
+        # Execute post-mount steps
+        echo "Executing post-mount steps..."
+        ${postMount}
+      ''}
+
+      umount mnt
+
+      (
+        cd layer
+        cmd='name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"'
+        find . -type c -exec bash -c "$cmd" \;
+      )
+
+      ${postUmount}
+      '');
+
+  exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }:
+    runWithOverlay {
+      inherit name fromImage fromImageName fromImageTag diskSize;
+
+      postMount = ''
+        echo "Packing raw image..."
+        tar -C mnt --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out .
+      '';
+    };
+
+
+  # Create an executable shell script which has the coreutils in its
+  # PATH. Since root scripts are executed in a blank environment, even
+  # things like `ls` or `echo` will be missing.
+  shellScript = name: text:
+    writeScript name ''
+      #!${stdenv.shell}
+      set -e
+      export PATH=${coreutils}/bin:/bin
+      ${text}
+    '';
+
+  nixRegistration = contents: runCommand "nix-registration" {
+    buildInputs = [ nixUnstable perl ];
+    # For obtaining the closure of `contents'.
+    exportReferencesGraph =
+      let contentsList = if builtins.isList contents then contents else [ contents ];
+      in map (x: [("closure-" + baseNameOf x) x]) contentsList;
+    }
+    ''
+      mkdir $out
+      printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out/db.dump
+      perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths
+    '';
+
+  # Create a "layer" (set of files).
+  mkPureLayer = {
+    # Name of the layer
+    name,
+    # JSON containing configuration and metadata for this layer.
+    baseJson,
+    # Files to add to the layer.
+    contents ? null,
+    # When copying the contents into the image, preserve symlinks to
+    # directories (see `rsync -K`).  Otherwise, transform those symlinks
+    # into directories.
+    keepContentsDirlinks ? false,
+    # Additional commands to run on the layer before it is tar'd up.
+    extraCommands ? "", uid ? 0, gid ? 0
+  }:
+    runCommand "docker-layer-${name}" {
+      inherit baseJson contents extraCommands;
+      buildInputs = [ jshon rsync ];
+    }
+    ''
+      mkdir layer
+      if [[ -n "$contents" ]]; then
+        echo "Adding contents..."
+        for item in $contents; do
+          echo "Adding $item"
+          rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
+        done
+      else
+        echo "No contents to add to layer."
+      fi
+
+      chmod ug+w layer
+
+      if [[ -n $extraCommands ]]; then
+        (cd layer; eval "$extraCommands")
+      fi
+
+      # Tar up the layer and throw it into 'layer.tar'.
+      echo "Packing layer..."
+      mkdir $out
+      tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
+
+      # Compute a checksum of the tarball.
+      echo "Computing layer checksum..."
+      tarsum=$(${tarsum} < $out/layer.tar)
+
+      # Add a 'checksum' field to the JSON, with the value set to the
+      # checksum of the tarball.
+      cat ${baseJson} | jshon -s "$tarsum" -i checksum > $out/json
+
+      # Indicate to docker that we're using schema version 1.0.
+      echo -n "1.0" > $out/VERSION
+
+      echo "Finished building layer '${name}'"
+    '';
+
+  # Make a "root" layer; required if we need to execute commands as a
+  # privileged user on the image. The commands themselves will be
+  # performed in a virtual machine sandbox.
+  mkRootLayer = {
+    # Name of the image.
+    name,
+    # Script to run as root. Bash.
+    runAsRoot,
+    # Files to add to the layer. If null, an empty layer will be created.
+    contents ? null,
+    # When copying the contents into the image, preserve symlinks to
+    # directories (see `rsync -K`).  Otherwise, transform those symlinks
+    # into directories.
+    keepContentsDirlinks ? false,
+    # JSON containing configuration and metadata for this layer.
+    baseJson,
+    # Existing image onto which to append the new layer.
+    fromImage ? null,
+    # Name of the image we're appending onto.
+    fromImageName ? null,
+    # Tag of the image we're appending onto.
+    fromImageTag ? null,
+    # How much disk to allocate for the temporary virtual machine.
+    diskSize ? 1024,
+    # Commands (bash) to run on the layer; these do not require sudo.
+    extraCommands ? ""
+  }:
+    # Generate an executable script from the `runAsRoot` text.
+    let runAsRootScript = shellScript "run-as-root.sh" runAsRoot;
+    in runWithOverlay {
+      name = "docker-layer-${name}";
+
+      inherit fromImage fromImageName fromImageTag diskSize;
+
+      preMount = lib.optionalString (contents != null && contents != []) ''
+        echo "Adding contents..."
+        for item in ${toString contents}; do
+          echo "Adding $item..."
+          rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/
+        done
+
+        chmod ug+w layer
+      '';
+
+      postMount = ''
+        mkdir -p mnt/{dev,proc,sys} mnt${storeDir}
+
+        # Mount /dev, /sys and the nix store as shared folders.
+        mount --rbind /dev mnt/dev
+        mount --rbind /sys mnt/sys
+        mount --rbind ${storeDir} mnt${storeDir}
+
+        # Execute the run as root script. See 'man unshare' for
+        # details on what's going on here; basically this command
+        # means that the runAsRootScript will be executed in a nearly
+        # completely isolated environment.
+        unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript}
+
+        # Unmount directories and remove them.
+        umount -R mnt/dev mnt/sys mnt${storeDir}
+        rmdir --ignore-fail-on-non-empty \
+          mnt/dev mnt/proc mnt/sys mnt${storeDir} \
+          mnt$(dirname ${storeDir})
+      '';
+
+      postUmount = ''
+        (cd layer; eval "${extraCommands}")
+
+        echo "Packing layer..."
+        mkdir $out
+        tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar .
+
+        # Compute the tar checksum and add it to the output json.
+        echo "Computing checksum..."
+        ts=$(${tarsum} < $out/layer.tar)
+        cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
+        # Indicate to docker that we're using schema version 1.0.
+        echo -n "1.0" > $out/VERSION
+
+        echo "Finished building layer '${name}'"
+      '';
+    };
+
+  # 1. extract the base image
+  # 2. create the layer
+  # 3. add layer deps to the layer itself, diffing with the base image
+  # 4. compute the layer id
+  # 5. put the layer in the image
+  # 6. repack the image
+  buildImage = args@{
+    # Image name.
+    name,
+    # Image tag.
+    tag ? "latest",
+    # Parent image, to append to.
+    fromImage ? null,
+    # Name of the parent image; will be read from the image otherwise.
+    fromImageName ? null,
+    # Tag of the parent image; will be read from the image otherwise.
+    fromImageTag ? null,
+    # Files to put on the image (a nix store path or list of paths).
+    contents ? null,
+    # When copying the contents into the image, preserve symlinks to
+    # directories (see `rsync -K`).  Otherwise, transform those symlinks
+    # into directories.
+    keepContentsDirlinks ? false,
+    # Docker config; e.g. what command to run on the container.
+    config ? null,
+    # Optional bash script to run on the files prior to fixturizing the layer.
+    extraCommands ? "", uid ? 0, gid ? 0,
+    # Optional bash script to run as root on the image when provisioning.
+    runAsRoot ? null,
+    # Size of the virtual machine disk to provision when building the image.
+    diskSize ? 1024,
+    # Time of creation of the image.
+    created ? "1970-01-01T00:00:01Z",
+  }:
+
+    let
+      baseName = baseNameOf name;
+
+      # Create a JSON blob of the configuration. Set the date to unix zero.
+      baseJson = writeText "${baseName}-config.json" (builtins.toJSON {
+        inherit created config;
+        architecture = "amd64";
+        os = "linux";
+      });
+
+      layer =
+        if runAsRoot == null
+        then mkPureLayer {
+          name = baseName;
+          inherit baseJson contents keepContentsDirlinks extraCommands uid gid;
+        } else mkRootLayer {
+          name = baseName;
+          inherit baseJson fromImage fromImageName fromImageTag
+                  contents keepContentsDirlinks runAsRoot diskSize
+                  extraCommands;
+        };
+      result = runCommand "docker-image-${baseName}.tar.gz" {
+        buildInputs = [ jshon pigz coreutils findutils jq ];
+        # Image name and tag must be lowercase
+        imageName = lib.toLower name;
+        imageTag = lib.toLower tag;
+        inherit fromImage baseJson;
+        layerClosure = writeReferencesToFile layer;
+        passthru.buildArgs = args;
+        passthru.layer = layer;
+      } ''
+        # Print tar contents:
+        # 1: Interpreted as relative to the root directory
+        # 2: With no trailing slashes on directories
+        # This is useful for ensuring that the output matches the
+        # values generated by the "find" command
+        ls_tar() {
+          for f in $(tar -tf $1 | xargs realpath -ms --relative-to=.); do
+            if [[ "$f" != "." ]]; then
+              echo "/$f"
+            fi
+          done
+        }
+
+        mkdir image
+        touch baseFiles
+        if [[ -n "$fromImage" ]]; then
+          echo "Unpacking base image..."
+          tar -C image -xpf "$fromImage"
+          # Do not import the base image configuration and manifest
+          chmod a+w image image/*.json
+          rm -f image/*.json
+
+          if [[ -z "$fromImageName" ]]; then
+            fromImageName=$(jshon -k < image/repositories|head -n1)
+          fi
+          if [[ -z "$fromImageTag" ]]; then
+            fromImageTag=$(jshon -e $fromImageName -k \
+                           < image/repositories|head -n1)
+          fi
+          parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
+                     < image/repositories)
+
+          for l in image/*/layer.tar; do
+            ls_tar $l >> baseFiles
+          done
+        fi
+
+        chmod -R ug+rw image
+
+        mkdir temp
+        cp ${layer}/* temp/
+        chmod ug+w temp/*
+
+        for dep in $(cat $layerClosure); do
+          find $dep >> layerFiles
+        done
+
+        echo "Adding layer..."
+        # Record the contents of the tarball with ls_tar.
+        ls_tar temp/layer.tar >> baseFiles
+
+        # Get the files in the new layer which were *not* present in
+        # the old layer, and record them as newFiles.
+        comm <(sort -n baseFiles|uniq) \
+             <(sort -n layerFiles|uniq|grep -v ${layer}) -1 -3 > newFiles
+        # Append the new files to the layer.
+        tar -rpf temp/layer.tar --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" \
+          --owner=0 --group=0 --no-recursion --files-from newFiles
+
+        echo "Adding meta..."
+
+        # If we have a parentID, add it to the json metadata.
+        if [[ -n "$parentID" ]]; then
+          cat temp/json | jshon -s "$parentID" -i parent > tmpjson
+          mv tmpjson temp/json
+        fi
+
+        # Take the sha256 sum of the generated json and use it as the layer ID.
+        # Compute the size and add it to the json under the 'Size' field.
+        layerID=$(sha256sum temp/json|cut -d ' ' -f 1)
+        size=$(stat --printf="%s" temp/layer.tar)
+        cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson
+        mv tmpjson temp/json
+
+        # Use the temp folder we've been working on to create a new image.
+        mv temp image/$layerID
+
+        # Create image json and image manifest
+        imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
+        manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
+        currentID=$layerID
+        while [[ -n "$currentID" ]]; do
+          layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1)
+          imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .")
+          imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .")
+          manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .")
+
+          currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true))
+        done
+
+        imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1)
+        echo "$imageJson" > "image/$imageJsonChecksum.json"
+        manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"")
+        echo "$manifestJson" > image/manifest.json
+
+        # Store the json under the name image/repositories.
+        jshon -n object \
+          -n object -s "$layerID" -i "$imageTag" \
+          -i "$imageName" > image/repositories
+
+        # Make the image read-only.
+        chmod -R a-w image
+
+        echo "Cooking the image..."
+        tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out
+
+        echo "Finished."
+      '';
+
+    in
+    result;
+
+  # Build an image and populate its nix database with the provided
+  # contents. The main purpose is to be able to use nix commands in
+  # the container.
+  # Be careful since this doesn't work well with multilayer.
+  buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }:
+    buildImage (args // {
+      extraCommands = ''
+        echo "Generating the nix database..."
+        echo "Warning: only the database of the deepest Nix layer is loaded."
+        echo "         If you want to use nix commands in the container, it would"
+        echo "         be better to only have one layer that contains a nix store."
+        # This requires Nix 1.12 or higher
+        export NIX_REMOTE=local?root=$PWD
+        ${nixUnstable}/bin/nix-store --load-db < ${nixRegistration contents}/db.dump
+
+        # We fill the store in order to run the 'verify' command that
+        # generates hash and size of output paths.
+        # Note when Nix 1.12 is be the stable one, the database dump
+        # generated by the exportReferencesGraph function will
+        # contains sha and size. See
+        # https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a
+        storePaths=$(cat ${nixRegistration contents}/storePaths)
+        echo "Copying everything to /nix/store (will take a while)..."
+        cp -prd $storePaths nix/store/
+        ${nixUnstable}/bin/nix-store --verify --check-contents
+
+        mkdir -p nix/var/nix/gcroots/docker/
+        for i in ${lib.concatStringsSep " " contents}; do
+          ln -s $i nix/var/nix/gcroots/docker/$(basename $i)
+        done;
+      '' + extraCommands;
+    });
+}
--- a/nixos/overlays/detjson.py
+++ b/nixos/overlays/detjson.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Deterministic layer json: https://github.com/docker/hub-feedback/issues/488
+
+import sys
+reload(sys)
+sys.setdefaultencoding('UTF8')
+import json
+
+# If any of the keys below are equal to a certain value
+# then we can delete it because it's the default value
+SAFEDELS = {
+    "Size": 0,
+    "config": {
+        "ExposedPorts": None,
+        "MacAddress": "",
+        "NetworkDisabled": False,
+        "PortSpecs": None,
+        "VolumeDriver": ""
+    }
+}
+SAFEDELS["container_config"] = SAFEDELS["config"]
+
+def makedet(j, safedels):
+    for k,v in safedels.items():
+        if k not in j:
+            continue
+        if type(v) == dict:
+            makedet(j[k], v)
+        elif j[k] == v:
+            del j[k]
+
+def main():
+    j = json.load(sys.stdin)
+    makedet(j, SAFEDELS)
+    json.dump(j, sys.stdout, sort_keys=True)
+
+if __name__ == '__main__':
+    main()
--- a/nixos/overlays/examples.nix
+++ b/nixos/overlays/examples.nix
@@ -0,0 +1,127 @@
+# Examples of using the docker tools to build packages.
+#
+# This file defines several docker images. In order to use an image,
+# build its derivation with `nix-build`, and then load the result with
+# `docker load`. For example:
+#
+#  $ nix-build '<nixpkgs>' -A dockerTools.examples.redis
+#  $ docker load < result
+
+{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }:
+
+rec {
+  # 1. basic example
+  bash = buildImage {
+    name = "bash";
+    contents = pkgs.bashInteractive;
+  };
+
+  # 2. service example, layered on another image
+  redis = buildImage {
+    name = "redis";
+    tag = "latest";
+
+    # for example's sake, we can layer redis on top of bash or debian
+    fromImage = bash;
+    # fromImage = debian;
+
+    contents = pkgs.redis;
+    runAsRoot = ''
+      mkdir -p /data
+    '';
+
+    config = {
+      Cmd = [ "/bin/redis-server" ];
+      WorkingDir = "/data";
+      Volumes = {
+        "/data" = {};
+      };
+    };
+  };
+
+  # 3. another service example
+  nginx = let
+    nginxPort = "80";
+    nginxConf = pkgs.writeText "nginx.conf" ''
+      user nginx nginx;
+      daemon off;
+      error_log /dev/stdout info;
+      pid /dev/null;
+      events {}
+      http {
+        access_log /dev/stdout;
+        server {
+          listen ${nginxPort};
+          index index.html;
+          location / {
+            root ${nginxWebRoot};
+          }
+        }
+      }
+    '';
+    nginxWebRoot = pkgs.writeTextDir "index.html" ''
+      <html><body><h1>Hello from NGINX</h1></body></html>
+    '';
+  in
+  buildImage {
+    name = "nginx-container";
+    contents = pkgs.nginx;
+
+    runAsRoot = ''
+      #!${pkgs.stdenv.shell}
+      ${shadowSetup}
+      groupadd --system nginx
+      useradd --system --gid nginx nginx
+    '';
+
+    config = {
+      Cmd = [ "nginx" "-c" nginxConf ];
+      ExposedPorts = {
+        "${nginxPort}/tcp" = {};
+      };
+    };
+  };
+
+  # 4. example of pulling an image. could be used as a base for other images
+  nixFromDockerHub = pullImage {
+    imageName = "nixos/nix";
+    imageTag = "1.11";
+    # this hash will need change if the tag is updated at docker hub
+    sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83";
+  };
+
+  # 5. example of multiple contents, emacs and vi happily coexisting
+  editors = buildImage {
+    name = "editors";
+    contents = [
+      pkgs.coreutils
+      pkgs.bash
+      pkgs.emacs
+      pkgs.vim
+      pkgs.nano
+    ];
+  };
+
+  # 6. nix example to play with the container nix store
+  # docker run -it --rm nix nix-store -qR $(nix-build '<nixpkgs>' -A nix)
+  nix = buildImageWithNixDb {
+    name = "nix";
+    contents = [
+      # nix-store uses cat program to display results as specified by
+      # the image env variable NIX_PAGER.
+      pkgs.coreutils
+      pkgs.nix
+    ];
+    config = {
+      Env = [ "NIX_PAGER=cat" ];
+    };
+  };
+
+  # 7. example of adding something on top of an image pull by our
+  # dockerTools chain.
+  onTopOfPulledImage = buildImage {
+    name = "onTopOfPulledImage";
+    fromImage = nixFromDockerHub;
+    contents = [ pkgs.hello ];
+  };
+}
--- a/nixos/overlays/kubernetes.nix
+++ b/nixos/overlays/kubernetes.nix
--- a/nixos/overlays/overlays.nix
+++ b/nixos/overlays/overlays.nix
@@ -0,0 +1,4 @@
+self: super: {
+  dockerTools = super.callPackage ./overlay/default.nix { go = self.go_1_9; };
+  # super.config.services.kubernetes = super.callPackage ./overlay/kubernetes.nix {};
+}
--- a/nixos/overlays/pull.nix
+++ b/nixos/overlays/pull.nix
@@ -0,0 +1,32 @@
+{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
+let
+  nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
+in
+# For simplicity we only support sha256.
+{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
+, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
+let
+  pullImage = vmTools.runInLinuxVM (
+    stdenv.mkDerivation {
+      inherit name imageId;
+
+      certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+      builder = ./pull.sh;
+
+      nativeBuildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
+
+      outputHashAlgo = "sha256";
+      outputHash = sha256;
+
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+
+      preVM = vmTools.createEmptyImage {
+        size = 2048;
+        fullName = "${name}-disk";
+      };
+
+      QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
+    });
+in
+  pullImage
--- a/nixos/overlays/pull.sh
+++ b/nixos/overlays/pull.sh
@@ -0,0 +1,36 @@
+source $stdenv/setup
+
+mkdir -p /var/lib/docker
+mkfs.ext4 /dev/vda
+mount -t ext4 /dev/vda /var/lib/docker
+
+modprobe virtio_net
+dhclient eth0
+
+mkdir -p /etc/ssl/certs/
+cp "$certs" "/etc/ssl/certs/"
+
+# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
+mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+cd /sys/fs/cgroup
+for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
+  mkdir -p $sys
+  if ! mountpoint -q $sys; then
+    if ! mount -n -t cgroup -o $sys cgroup $sys; then
+      rmdir $sys || true
+    fi
+  fi
+done
+
+# run docker daemon
+dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
+
+until docker ps 2>/dev/null; do
+  printf '.'
+  sleep 1
+done
+
+rm -r $out
+
+docker pull ${imageId}
+docker save ${imageId} > $out
--- a/nixos/overlays/tarsum.go
+++ b/nixos/overlays/tarsum.go
@@ -0,0 +1,24 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"github.com/docker/docker/pkg/tarsum"
+)
+
+func main() {
+	ts, err := tarsum.NewTarSum(os.Stdin, true, tarsum.Version1)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	if _, err = io.Copy(ioutil.Discard, ts); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	fmt.Println(ts.Sum(nil))
+}
--- a/nixos/packages.nix
+++ b/nixos/packages.nix
--- a/nixos/users.nix
+++ b/nixos/users.nix