From 0ea0abd3161555d928b56b53b1bb36b5620013cf Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Tue, 17 Apr 2018 21:35:56 +0200 Subject: [PATCH] Major revamp. --- base/pki.nix | 124 -- bin/teardown.sh | 2 +- fs0.nix | 63 + git.nix | 17 - hardware-configuration/fs0-0.nix | 29 + hardware-configuration/fs0-1.nix | 26 + .../k0-0.nix | 0 .../k0-1.nix | 0 .../k0-2.nix | 0 .../k0-3.nix | 0 hardware-configuration/k0-4.nix | 21 + hardware-configuration/k0-5.nix | 21 + hardware-configuration/k1-0.nix | 21 + hardware-configuration/k1-1.nix | 21 + k0.nix | 74 ++ k8s.nix | 209 --- certs.nix => lib/certs.nix | 0 lib/k8s.nix | 173 +++ lib/pki.nix | 124 ++ {base => nixos}/configuration.nix | 15 +- {base => nixos}/desktop.nix | 0 {base => nixos}/nix-home.nix | 0 nixos/overlays/default.nix | 593 +++++++++ nixos/overlays/detjson.py | 40 + nixos/overlays/examples.nix | 127 ++ nixos/overlays/kubernetes.nix | 1157 +++++++++++++++++ nixos/overlays/overlays.nix | 4 + nixos/overlays/pull.nix | 32 + nixos/overlays/pull.sh | 36 + nixos/overlays/tarsum.go | 24 + {base => nixos}/packages.nix | 0 {base => nixos}/users.nix | 0 32 files changed, 2600 insertions(+), 353 deletions(-) delete mode 100644 base/pki.nix create mode 100644 fs0.nix delete mode 100644 git.nix create mode 100644 hardware-configuration/fs0-0.nix create mode 100644 hardware-configuration/fs0-1.nix rename hw/k8s0-0.nix => hardware-configuration/k0-0.nix (100%) rename hw/k8s0-1.nix => hardware-configuration/k0-1.nix (100%) rename hw/k8s0-2.nix => hardware-configuration/k0-2.nix (100%) rename hw/k8s0-3.nix => hardware-configuration/k0-3.nix (100%) create mode 100644 hardware-configuration/k0-4.nix create mode 100644 hardware-configuration/k0-5.nix create mode 100644 hardware-configuration/k1-0.nix create mode 100644 hardware-configuration/k1-1.nix create mode 100644 k0.nix delete mode 100644 k8s.nix rename certs.nix => lib/certs.nix (100%) create mode 100644 lib/k8s.nix create mode 100644 lib/pki.nix rename {base => nixos}/configuration.nix (76%) rename {base => nixos}/desktop.nix (100%) rename {base => nixos}/nix-home.nix (100%) create mode 100644 nixos/overlays/default.nix create mode 100644 nixos/overlays/detjson.py create mode 100644 nixos/overlays/examples.nix create mode 100644 nixos/overlays/kubernetes.nix create mode 100644 nixos/overlays/overlays.nix create mode 100644 nixos/overlays/pull.nix create mode 100644 nixos/overlays/pull.sh create mode 100644 nixos/overlays/tarsum.go rename {base => nixos}/packages.nix (100%) rename {base => nixos}/users.nix (100%) diff --git a/base/pki.nix b/base/pki.nix deleted file mode 100644 index 4fa59c5..0000000 --- a/base/pki.nix +++ /dev/null @@ -1,124 +0,0 @@ -with import {}; -let - ca-config = pkgs.writeText "ca-config.json" '' - { - "signing": { - "default": { - "expiry": "43800h" - }, - "profiles": { - "server": { - "expiry": "43800h", - "usages": [ - "signing", - "key encipherment", - "server auth" - ] - }, - "client": { - "expiry": "43800h", - "usages": [ - "signing", - "key encipherment", - "client auth" - ] - } - } - } - } - ''; - - csr = args: pkgs.writeText "${args.cn}-cert.json" '' - { - "CN": "${args.cn}", - "hosts": [ ${args.hosts} ], - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "NO", - "L": "Tromsø", - "O": "Serit IT Partner Tromsø AS", - "OU": "", - "ST": "" - } - ] - } - ''; - - ca-csr = csr { cn = "kubernetes"; hosts = ""; }; - ca = pkgs.runCommand "ca-cert" { - buildInputs = [ pkgs.cfssl ]; - } '' cfssl genkey -initca ${ca-csr} | cfssljson -bare ca; \ - mkdir -p $out; cp *.pem $out''; - - ca_cert = "${ca}/ca.pem"; - ca_key = "${ca}/ca-key.pem"; - - cfssl = name: profile: '' - cfssl gencert -ca ${ca_cert} -ca-key ${ca_key} \ - -config=${ca-config} -profile=${profile} ${name} | cfssljson -bare cert; \ - mkdir -p $out; cp *.pem $out - ''; -in - rec { - inherit ca_cert; - inherit ca_key; - inherit csr; - - mkCert = cert: - pkgs.runCommand "${cert.name}-cert" { - buildInputs = [ pkgs.cfssl ]; - } (cfssl cert.csr cert.profile); - - # server-cert = mkCert { - # name = "kubernetes"; - # csr = csr { - # cn = "kubernetes"; - # hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"''; - # }; - # profile = "server"; - # }; - - # etcd0-cert = mkCert { - # name = "etcd0"; - # csr = csr { - # cn = "etcd0"; - # hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"''; - # }; - # profile = "peer"; - # }; - - # etcd1-cert = mkCert { - # name = "etcd1"; - # csr = csr { - # cn = "etcd1"; - # hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"''; - # }; - # profile = "peer"; - # }; - - # client-cert = mkCert { - # name = "client"; - # csr = csr { - # cn = "client"; - # hosts = ''''; - # }; - # profile = "client"; - # }; - - # server_key = "${server-cert}/cert-key.pem"; - # server_cert = "${server-cert}/cert.pem"; - - # etcd0_key = "${etcd0-cert}/cert-key.pem"; - # etcd0_cert = "${etcd0-cert}/cert.pem"; - - # etcd1_key = "${etcd1-cert}/cert-key.pem"; - # etcd1_cert = "${etcd1-cert}/cert.pem"; - - # client_key = "${client-cert}/cert-key.pem"; - # client_cert = "${client-cert}/cert.pem"; - - } diff --git a/bin/teardown.sh b/bin/teardown.sh index 6320984..839645e 100755 --- a/bin/teardown.sh +++ b/bin/teardown.sh @@ -17,6 +17,6 @@ nixops modify -d $d $f nixops deploy -d $d rm $f +nixops reboot -d $d nixops ssh-for-each -d $d "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd" -nixops ssh-for-each -d $d reboot diff --git a/fs0.nix b/fs0.nix new file mode 100644 index 0000000..7935110 --- /dev/null +++ b/fs0.nix @@ -0,0 +1,63 @@ +with import {}; +let + pki = pkgs.callPackage ./lib/pki.nix {}; + certs = { + ca = pki.ca; + fs = pki.etcd '' + "fs0-0", + "fs0-1", + "10.253.18.106", + "10.1.2.164", + "127.0.0.1" + ''; + }; + clusterHosts = '' + 10.253.18.106 fs0-0 + 10.1.2.164 fs0-1 + ''; + + nixosConfig = node: { + imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ]; + networking = { + hostName = node; + extraHosts = clusterHosts; + # firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; + # firewall.allowedTCPPorts = [ 80 443 ]; + }; + environment.systemPackages = [ pkgs.tshark ]; + services.glusterfs = { + enable = true; + tlsSettings = { + caCert = certs.ca.cert; + tlsKeyPath = certs.fs.key; + tlsPem = certs.fs.cert; + }; + }; + }; +in +{ + fs0-0 = { ... }: + let + base = nixosConfig "fs0-0"; + in + { + deployment.targetHost = "10.253.18.106"; + require = [ base ]; + services.nfs.server = { + enable=true; + exports= '' + /data/vol1 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + networking.firewall.allowedTCPPorts = [ 111 2049 ]; + networking.firewall.allowedUDPPorts = [ 111 2049 ]; + }; + fs0-1 = { ... }: + let + base = nixosConfig "fs0-1"; + in + { + deployment.targetHost = "10.1.2.164"; + require = [ base ]; + }; +} diff --git a/git.nix b/git.nix deleted file mode 100644 index 348b04f..0000000 --- a/git.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ - git01 = { config, lib, pkgs, ... }: - { - deployment.targetHost = "10.253.18.103"; - networking.hostName = "git01"; # Define your hostname - imports = [ ./hw/git01.nix ./base/configuration.nix ]; - services.nfs.server = { - enable=true; - exports= '' - /data 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - ''; - }; - networking.firewall.allowedTCPPorts = [2049 111 20048]; - networking.firewall.allowedUDPPorts = [2049 111 20048]; - }; -} diff --git a/hardware-configuration/fs0-0.nix b/hardware-configuration/fs0-0.nix new file mode 100644 index 0000000..655171f --- /dev/null +++ b/hardware-configuration/fs0-0.nix @@ -0,0 +1,29 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/6b3d4c49-9719-49b3-8210-d53374cd0eff"; + fsType = "ext4"; + }; + fileSystems."/var/log" = + { device = "/dev/disk/by-uuid/c1e78683-4fde-4029-a9f3-7631df649b2f"; + fsType = "ext4"; + }; + fileSystems."/data" = + { device = "/dev/sdb1"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/hardware-configuration/fs0-1.nix b/hardware-configuration/fs0-1.nix new file mode 100644 index 0000000..e7cc1f4 --- /dev/null +++ b/hardware-configuration/fs0-1.nix @@ -0,0 +1,26 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/e8820516-9c21-46f4-9dde-a7a77bf67bbd"; + fsType = "ext4"; + }; + fileSystems."/var/log" = + { device = "/dev/disk/by-uuid/c130b88c-0699-4836-b967-47bdee0a5453"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} + diff --git a/hw/k8s0-0.nix b/hardware-configuration/k0-0.nix similarity index 100% rename from hw/k8s0-0.nix rename to hardware-configuration/k0-0.nix diff --git a/hw/k8s0-1.nix b/hardware-configuration/k0-1.nix similarity index 100% rename from hw/k8s0-1.nix rename to hardware-configuration/k0-1.nix diff --git a/hw/k8s0-2.nix b/hardware-configuration/k0-2.nix similarity index 100% rename from hw/k8s0-2.nix rename to hardware-configuration/k0-2.nix diff --git a/hw/k8s0-3.nix b/hardware-configuration/k0-3.nix similarity index 100% rename from hw/k8s0-3.nix rename to hardware-configuration/k0-3.nix diff --git a/hardware-configuration/k0-4.nix b/hardware-configuration/k0-4.nix new file mode 100644 index 0000000..cb8e0f1 --- /dev/null +++ b/hardware-configuration/k0-4.nix @@ -0,0 +1,21 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/241cdf57-a2b5-482d-a522-01725badc859"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/hardware-configuration/k0-5.nix b/hardware-configuration/k0-5.nix new file mode 100644 index 0000000..fe4ae9d --- /dev/null +++ b/hardware-configuration/k0-5.nix @@ -0,0 +1,21 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/67441b95-19f2-484d-b57b-3f4b2a55f3cc"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/hardware-configuration/k1-0.nix b/hardware-configuration/k1-0.nix new file mode 100644 index 0000000..5e5d1e2 --- /dev/null +++ b/hardware-configuration/k1-0.nix @@ -0,0 +1,21 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/2e7ba83d-014f-4ef5-a1ce-fc9e34ce7b83"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/hardware-configuration/k1-1.nix b/hardware-configuration/k1-1.nix new file mode 100644 index 0000000..910e695 --- /dev/null +++ b/hardware-configuration/k1-1.nix @@ -0,0 +1,21 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, ... }: + +{ + imports = [ ]; + + boot.initrd.availableKernelModules = [ "ata_piix" "mptspi" "floppy" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/70b9d730-9cb6-48e2-8e00-fa78c8feefdf"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + nix.maxJobs = lib.mkDefault 1; +} diff --git a/k0.nix b/k0.nix new file mode 100644 index 0000000..27b79b3 --- /dev/null +++ b/k0.nix @@ -0,0 +1,74 @@ +with import {}; +let + pki = pkgs.callPackage ./lib/pki.nix {}; + certs = { + ca = pki.ca; + apiserver = pki.apiserver '' + "10.253.18.100", + "10.0.0.1", + "127.0.0.1", + "kubernetes", + "etcd0", + "k0-0" + ''; + kube-proxy = pki.kube-proxy; + admin = pki.admin; + etcd = pki.etcd '' + "etcd0", + "etcd1", + "etcd2", + "10.253.18.100", + "10.253.18.101", + "10.253.18.102", + "127.0.0.1" + ''; + k0-0 = pki.worker { name = "k0-0"; ip = "10.253.18.100"; }; + k0-1 = pki.worker { name = "k0-1"; ip = "10.253.18.101"; }; + k0-2 = pki.worker { name = "k0-2"; ip = "10.253.18.102"; }; + k0-3 = pki.worker { name = "k0-3"; ip = "10.253.18.103"; }; + k0-4 = pki.worker { name = "k0-4"; ip = "10.253.18.107"; }; + k0-5 = pki.worker { name = "k0-5"; ip = "10.253.18.108"; }; + }; + cluster = callPackage ./lib/k8s.nix { + kubeMaster = "10.253.18.100"; + etcdNodes = [ "etcd0" "etcd1" "etcd2" ]; + clusterHosts = '' + 10.253.18.100 k0-0 etcd0 kubernetes + 10.253.18.101 k0-1 etcd1 + 10.253.18.102 k0-2 etcd2 + 10.253.18.103 k0-3 + 10.253.18.107 k0-4 + 10.253.18.108 k0-5 + 10.253.18.106 fs0-0 + 10.1.2.164 fs0-1 + ''; + inherit certs; + }; +in +{ + k0-0 = cluster.apiserver "k0-0" "10.253.18.100" "etcd0"; + k0-1 = cluster.server "k0-1" "10.253.18.101" "etcd1"; + k0-2 = cluster.server "k0-2" "10.253.18.102" "etcd2"; + k0-3 = cluster.worker "k0-3" "10.253.18.103"; + k0-4 = cluster.worker "k0-4" "10.253.18.107"; + k0-5 = cluster.worker "k0-5" "10.253.18.108"; + + # k0-3 = { config, lib, pkgs, ... }: + # let + # instance = "k0-3"; + # base = k8s.nixosConfig instance; + # in + # { + # deployment.targetHost = "10.253.18.103"; + # require = [ k8s.base (k8s.kubeConfig instance) (k8s.kubeNode instance) ]; + # services.kubernetes.addons.dns.enable = false; + # services.nfs.server = { + # enable=true; + # exports= '' + # /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + # ''; + # }; + # networking.firewall.allowedTCPPorts = [ 111 2049 ]; + # networking.firewall.allowedUDPPorts = [ 111 2049 ]; + # }; +} diff --git a/k8s.nix b/k8s.nix deleted file mode 100644 index 76a4ebc..0000000 --- a/k8s.nix +++ /dev/null @@ -1,209 +0,0 @@ -with import ./certs.nix; -let - pkgs = import {}; - - kube_apiserver = "https://10.253.18.100:443"; - etcdServers = [ "etcd0" "etcd1" "etcd2" ]; - etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers; - etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers; - - etcdConfig = name: { - services.etcd = { - inherit name; - enable = true; - listenClientUrls = ["https://0.0.0.0:2379"]; - listenPeerUrls = ["https://0.0.0.0:2380"]; - peerClientCertAuth = true; - keyFile = etcd_key; - certFile = etcd_cert; - trustedCaFile = ca_pem; - advertiseClientUrls = [ "https://${name}:2379" ]; - initialAdvertisePeerUrls = [ "https://${name}:2380" ]; - initialCluster = etcdCluster; - }; - environment.variables = { - ETCDCTL_KEY_FILE = "${etcd_client_key}"; - ETCDCTL_CERT_FILE = "${etcd_client_cert}"; - ETCDCTL_CA_FILE = "${ca_pem}"; - ETCDCTL_PEERS = "https://127.0.0.1:2379"; - }; - networking.firewall.allowedTCPPorts = [ 2379 2380 ]; - systemd.services.flannel.after = [ "etcd.service" ]; - }; - - kubeconfig = { - caFile = ca_pem; - keyFile = worker_key; - certFile = worker_cert; - server = kube_apiserver; - }; - - kubeNode = { - services.kubernetes = { - roles = [ "node" ]; - kubeconfig = { - server = kube_apiserver; - keyFile = worker_key; - certFile = worker_cert; - caFile = ca_pem; - }; - kubelet = { - enable = true; - clientCaFile = ca_pem; - tlsKeyFile = worker_key; - tlsCertFile = worker_cert; - networkPlugin = null; - # clusterDns = "10.253.18.100"; - clusterDns = "10.0.0.254"; - inherit kubeconfig; - }; - }; - networking = { - firewall = { - enable = true; - # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; - allowedTCPPorts = [ 53 4194 10250 ]; - allowedUDPPorts = [ 53 ]; - extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; - }; - }; - virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; - }; - - kubeMaster = { - services.kubernetes = { - roles = [ "master" ]; - kubelet.unschedulable = true; - apiserver = { - address = "0.0.0.0"; - publicAddress = "0.0.0.0"; - advertiseAddress = "10.253.18.100"; - securePort = 443; - tlsKeyFile = apiserver_key; - tlsCertFile = apiserver_cert; - clientCaFile = ca_pem; - kubeletClientCaFile = ca_pem; - kubeletClientKeyFile = worker_key; - kubeletClientCertFile = worker_cert; - serviceAccountKeyFile = apiserver_key; - }; - scheduler.leaderElect = true; - controllerManager = { - leaderElect = true; - serviceAccountKeyFile = apiserver_key; - rootCaFile = ca_pem; - inherit kubeconfig; - }; - addons.dashboard.enable = true; - addons.dns.enable = true; - }; - networking.firewall = { - allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ]; - # allowedUDPPorts = [ 4053 ]; - }; - environment.systemPackages = [ pkgs.kubernetes-helm ]; - }; - - kubeConfig = { - services.kubernetes = { - verbose = false; - caFile = ca_pem; - flannel.enable = true; - clusterCidr = "10.10.0.0/16"; - etcd = { - servers = etcdEndpoints; - keyFile = etcd_client_key; - certFile = etcd_client_cert; - caFile = ca_pem; - }; - proxy = { - inherit kubeconfig; - }; - }; - }; - - baseConfig = node: { - imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; - networking = { - hostName = node; - extraHosts = '' - 10.253.18.100 etcd0 kubernetes - 10.253.18.101 etcd1 - 10.253.18.102 etcd2 - ''; - firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; - firewall.allowedTCPPorts = [ 80 443 ]; - }; - environment.systemPackages = [ pkgs.tshark ]; - }; - - minion = host: ip: { config, lib, pkgs, ... }: - let - inherit host; - base = baseConfig host; - in - { - deployment.targetHost = ip; - require = [ base kubeConfig kubeNode ]; - services.kubernetes.addons.dns.enable = false; - }; -in -{ - k8s0-0 = { config, lib, pkgs, ... }: - let - base = baseConfig "k8s0-0"; - etcd = etcdConfig "etcd0"; - in - { - deployment.targetHost = "10.253.18.100"; - require = [ base etcd kubeConfig kubeMaster kubeNode ]; - services.dockerRegistry = { - enable = true; - listenAddress = "0.0.0.0"; - extraConfig = { - REGISTRY_HTTP_TLS_CERTIFICATE = "${apiserver_cert}"; - REGISTRY_HTTP_TLS_KEY = "${apiserver_key}"; - }; - }; - }; - - k8s0-1 = { config, lib, pkgs, ... }: - let - base = baseConfig "k8s0-1"; - etcd = etcdConfig "etcd1"; - in - { - deployment.targetHost = "10.253.18.101"; - require = [ base etcd kubeConfig kubeNode ]; - services.kubernetes.addons.dns.enable = false; - }; - - k8s0-2 = { config, lib, pkgs, ... }: - let - base = baseConfig "k8s0-2"; - etcd = etcdConfig "etcd2"; - in - { - deployment.targetHost = "10.253.18.102"; - require = [ base etcd kubeConfig kubeNode ]; - services.kubernetes.addons.dns.enable = false; - }; - - k8s0-3 = { config, lib, pkgs, ... }: - let - base = baseConfig "k8s0-3"; - in - { - deployment.targetHost = "10.253.18.103"; - require = [ base kubeConfig kubeNode ]; - services.kubernetes.addons.dns.enable = false; - services.nfs.server = { - enable=true; - exports= '' - /vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - ''; - }; - networking.firewall.allowedTCPPorts = [ 111 2049 ]; - networking.firewall.allowedUDPPorts = [ 111 2049 ]; - }; -} diff --git a/certs.nix b/lib/certs.nix similarity index 100% rename from certs.nix rename to lib/certs.nix diff --git a/lib/k8s.nix b/lib/k8s.nix new file mode 100644 index 0000000..20345bb --- /dev/null +++ b/lib/k8s.nix @@ -0,0 +1,173 @@ +{ pkgs, kubeMaster, etcdNodes, clusterHosts, certs, ...}: +let + kubeApiserver = "https://${kubeMaster}:443"; + localApiserver = "https://127.0.0.1:8080"; + etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes; + etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes; +in +rec { + etcdConfig = name: { + services.etcd = { + inherit name; + enable = true; + listenClientUrls = ["https://0.0.0.0:2379"]; + listenPeerUrls = ["https://0.0.0.0:2380"]; + peerClientCertAuth = true; + keyFile = certs.etcd.key; + certFile = certs.etcd.cert; + trustedCaFile = certs.ca.cert; + advertiseClientUrls = [ "https://${name}:2379" ]; + initialAdvertisePeerUrls = [ "https://${name}:2380" ]; + initialCluster = etcdCluster; + }; + environment.variables = { + ETCDCTL_KEY_FILE = "${certs.admin.key}"; + ETCDCTL_CERT_FILE = "${certs.admin.cert}"; + ETCDCTL_CA_FILE = "${certs.ca.cert}"; + ETCDCTL_PEERS = "https://127.0.0.1:2379"; + }; + networking.firewall.allowedTCPPorts = [ 2379 2380 ]; + systemd.services.flannel.after = [ "etcd.service" ]; + }; + + clientConf = instance: { + server = kubeApiserver; + keyFile = certs.${instance}.key; + certFile = certs.${instance}.cert; + caFile = certs.ca.cert; + }; + + kubeNode = instance: { + services.kubernetes = rec { + roles = [ "node" ]; + kubeconfig = clientConf instance; + kubelet = { + enable = true; + clientCaFile = certs.ca.cert; + tlsKeyFile = certs.${instance}.key; + tlsCertFile = certs.${instance}.cert; + networkPlugin = null; + clusterDns = "10.0.0.254"; + extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice"; + inherit kubeconfig; + }; + }; + networking = { + firewall = { + enable = true; + # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; + allowedTCPPorts = [ 53 4194 10250 ]; + allowedUDPPorts = [ 53 ]; + extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; + }; + }; + virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8"; + # systemd.services.kube-proxy.path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod]; + }; + + kubeMaster = { + services.kubernetes = { + roles = [ "master" ]; + kubelet.unschedulable = true; + apiserver = { + address = kubeMaster; + advertiseAddress = kubeMaster; + authorizationMode = [ "Node" "RBAC" ]; + securePort = 443; + tlsKeyFile = certs.apiserver.key; + tlsCertFile = certs.apiserver.cert; + clientCaFile = certs.ca.cert; + kubeletClientCaFile = certs.ca.cert; + kubeletClientKeyFile = certs.apiserver.key; + kubeletClientCertFile = certs.apiserver.cert; + serviceAccountKeyFile = certs.apiserver.key; + }; + scheduler.leaderElect = true; + controllerManager = { + leaderElect = true; + serviceAccountKeyFile = certs.apiserver.key; + rootCaFile = certs.ca.cert; + kubeconfig.server = localApiserver; + }; + scheduler.kubeconfig.server = localApiserver; + addons.dashboard.enable = true; + addons.dns.enable = true; + }; + networking.firewall = { + allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ]; + # allowedUDPPorts = [ 4053 ]; + }; + environment.systemPackages = [ pkgs.kubernetes-helm ]; + }; + + kubeConfig = instance: { + services.kubernetes = { + verbose = false; + caFile = certs.ca.cert; + flannel.enable = true; + clusterCidr = "10.10.0.0/16"; + etcd = { + servers = etcdEndpoints; + keyFile = certs.apiserver.key; + certFile = certs.apiserver.cert; + caFile = certs.ca.cert; + }; + proxy = { + kubeconfig = clientConf "kube-proxy"; + }; + }; + }; + + nixosConfig = node: { + imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ]; + networking = { + hostName = node; + extraHosts = clusterHosts; + firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ]; + firewall.allowedTCPPorts = [ 80 443 ]; + }; + environment.systemPackages = [ pkgs.tshark ]; + }; + + worker = host: ip: { config, lib, pkgs, ... }: + let + instance = host; + base = nixosConfig host; + in + { + deployment.targetHost = ip; + require = [ base (kubeConfig instance) (kubeNode instance) ]; + services.kubernetes.addons.dns.enable = false; + }; + + server = host: etc: ip: { config, lib, pkgs, ... }: + let + instance = host; + base = nixosConfig instance; + etcd = etcdConfig etc; + in + { + deployment.targetHost = ip; + require = [ base etcd (kubeConfig instance) (kubeNode instance) ]; + services.kubernetes.addons.dns.enable = false; + }; + + apiserver = host: ip: etc: { config, lib, pkgs, ... }: + let + instance = host; + base = nixosConfig instance; + etcd = etcdConfig etc; + in + { + deployment.targetHost = ip; + require = [ base etcd (kubeConfig instance) kubeMaster (kubeNode instance) ]; + services.dockerRegistry = { + enable = true; + listenAddress = "0.0.0.0"; + extraConfig = { + REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}"; + REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}"; + }; + }; + }; +} diff --git a/lib/pki.nix b/lib/pki.nix new file mode 100644 index 0000000..793d750 --- /dev/null +++ b/lib/pki.nix @@ -0,0 +1,124 @@ +{ pkgs ? import {} }: rec { + ca-config = pkgs.writeText "ca-config.json" '' + { + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "kubernetes": { + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ], + "expiry": "8760h" + } + } + } + } + ''; + + gencsr = args: pkgs.writeText "${args.name}-csr.json" '' + { + "CN": "${args.cn}", + "hosts": [ ${args.hosts} ], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "O": "${args.o}" + } + ] + } + ''; + + initca = + let + ca_csr = gencsr { + name = "kubernetes"; + cn = "kubernetes"; + o = "kubernetes"; + hosts = ""; + }; + in + pkgs.runCommand "ca" { + buildInputs = [ pkgs.cfssl ]; + } '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \ + mkdir -p $out; cp *.pem $out''; + ca = { + key = "${initca}/ca-key.pem"; + cert = "${initca}/ca.pem"; + }; + + cfssl = conf: '' + cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \ + -config=${ca-config} -profile=kubernetes ${conf.csr} | \ + cfssljson -bare cert; \ + mkdir -p $out; cp *.pem $out + ''; + + gencert = conf: + let + drv = pkgs.runCommand "${conf.name}" { + buildInputs = [ pkgs.cfssl ]; + } (cfssl conf); + in + { + key = "${drv}/cert-key.pem"; + cert = "${drv}/cert.pem"; + }; + + admin = gencert rec { + name = "admin"; + csr = gencsr { + inherit name; + cn = "admin"; + o = "system:masters"; + hosts = ""; + }; + }; + + apiserver = hosts: + gencert rec { + name = "kubernetes"; + csr = gencsr { + inherit name hosts; + cn = "kubernetes"; + o = "kubernetes"; + }; + }; + + etcd = hosts: gencert rec { + name = "etcd"; + csr = gencsr { + inherit name hosts; + cn = "etcd"; + o = "kubernetes"; + }; + }; + + kube-proxy = gencert rec { + name = "kube-proxy"; + csr = gencsr { + inherit name; + cn = "system:kube-proxy"; + o = "system:node-proxier"; + hosts = ""; + }; + }; + + worker = instance: + gencert rec { + name = instance.name; + csr = gencsr { + inherit name; + cn = "system:node:${instance.name}"; + o = "system:nodes"; + hosts = ''"${instance.name}","${instance.ip}"''; + }; + }; +} diff --git a/base/configuration.nix b/nixos/configuration.nix similarity index 76% rename from base/configuration.nix rename to nixos/configuration.nix index b40d54a..c99180c 100644 --- a/base/configuration.nix +++ b/nixos/configuration.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, lib, ... }: { # Use the GRUB 2 boot loader. boot.loader.grub.enable = true; @@ -29,5 +29,16 @@ security.rtkit.enable = true; - imports = [ ./users.nix ./packages.nix ]; + disabledModules = [ "services/cluster/kubernetes/default.nix" ]; + + imports = [ + ./users.nix + ./packages.nix + ./overlays/kubernetes.nix + ]; + + nixpkgs.overlays = [ + (import ./overlays/overlays.nix) + ]; + } diff --git a/base/desktop.nix b/nixos/desktop.nix similarity index 100% rename from base/desktop.nix rename to nixos/desktop.nix diff --git a/base/nix-home.nix b/nixos/nix-home.nix similarity index 100% rename from base/nix-home.nix rename to nixos/nix-home.nix diff --git a/nixos/overlays/default.nix b/nixos/overlays/default.nix new file mode 100644 index 0000000..b8eda3d --- /dev/null +++ b/nixos/overlays/default.nix @@ -0,0 +1,593 @@ +{ + callPackage, + coreutils, + docker, + e2fsprogs, + findutils, + go, + jshon, + jq, + lib, + pkgs, + pigz, + nixUnstable, + perl, + runCommand, + rsync, + shadow, + stdenv, + storeDir ? builtins.storeDir, + utillinux, + vmTools, + writeReferencesToFile, + writeScript, + writeText, +}: + +# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future. + +rec { + + examples = import ./examples.nix { + inherit pkgs buildImage pullImage shadowSetup buildImageWithNixDb; + }; + + pullImage = callPackage ./pull.nix {}; + + # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash. + # And we cannot untar it, because then we cannot preserve permissions ecc. + tarsum = runCommand "tarsum" { + buildInputs = [ go ]; + } '' + mkdir tarsum + cd tarsum + + cp ${./tarsum.go} tarsum.go + export GOPATH=$(pwd) + mkdir -p src/github.com/docker/docker/pkg + ln -sT ${docker.src}/components/engine/pkg/tarsum src/github.com/docker/docker/pkg/tarsum + go build + + cp tarsum $out + ''; + + # buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM + mergeDrvs = { + derivations, + onlyDeps ? false + }: + runCommand "merge-drvs" { + inherit derivations onlyDeps; + } '' + if [[ -n "$onlyDeps" ]]; then + echo $derivations > $out + exit 0 + fi + + mkdir $out + for derivation in $derivations; do + echo "Merging $derivation..." + if [[ -d "$derivation" ]]; then + # If it's a directory, copy all of its contents into $out. + cp -drf --preserve=mode -f $derivation/* $out/ + else + # Otherwise treat the derivation as a tarball and extract it + # into $out. + tar -C $out -xpf $drv || true + fi + done + ''; + + # Helper for setting up the base files for managing users and + # groups, only if such files don't exist already. It is suitable for + # being used in a runAsRoot script. + shadowSetup = '' + export PATH=${shadow}/bin:$PATH + mkdir -p /etc/pam.d + if [[ ! -f /etc/passwd ]]; then + echo "root:x:0:0::/root:${stdenv.shell}" > /etc/passwd + echo "root:!x:::::::" > /etc/shadow + fi + if [[ ! -f /etc/group ]]; then + echo "root:x:0:" > /etc/group + echo "root:x::" > /etc/gshadow + fi + if [[ ! -f /etc/pam.d/other ]]; then + cat > /etc/pam.d/other </dev/null || true)) + done + + mkdir work + mkdir layer + mkdir mnt + + ${lib.optionalString (preMount != "") '' + # Execute pre-mount steps + echo "Executing pre-mount steps..." + ${preMount} + ''} + + if [ -n "$lowerdir" ]; then + mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt + else + mount --bind layer mnt + fi + + ${lib.optionalString (postMount != "") '' + # Execute post-mount steps + echo "Executing post-mount steps..." + ${postMount} + ''} + + umount mnt + + ( + cd layer + cmd='name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"' + find . -type c -exec bash -c "$cmd" \; + ) + + ${postUmount} + ''); + + exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }: + runWithOverlay { + inherit name fromImage fromImageName fromImageTag diskSize; + + postMount = '' + echo "Packing raw image..." + tar -C mnt --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out . + ''; + }; + + + # Create an executable shell script which has the coreutils in its + # PATH. Since root scripts are executed in a blank environment, even + # things like `ls` or `echo` will be missing. + shellScript = name: text: + writeScript name '' + #!${stdenv.shell} + set -e + export PATH=${coreutils}/bin:/bin + ${text} + ''; + + nixRegistration = contents: runCommand "nix-registration" { + buildInputs = [ nixUnstable perl ]; + # For obtaining the closure of `contents'. + exportReferencesGraph = + let contentsList = if builtins.isList contents then contents else [ contents ]; + in map (x: [("closure-" + baseNameOf x) x]) contentsList; + } + '' + mkdir $out + printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out/db.dump + perl ${pkgs.pathsFromGraph} closure-* > $out/storePaths + ''; + + # Create a "layer" (set of files). + mkPureLayer = { + # Name of the layer + name, + # JSON containing configuration and metadata for this layer. + baseJson, + # Files to add to the layer. + contents ? null, + # When copying the contents into the image, preserve symlinks to + # directories (see `rsync -K`). Otherwise, transform those symlinks + # into directories. + keepContentsDirlinks ? false, + # Additional commands to run on the layer before it is tar'd up. + extraCommands ? "", uid ? 0, gid ? 0 + }: + runCommand "docker-layer-${name}" { + inherit baseJson contents extraCommands; + buildInputs = [ jshon rsync ]; + } + '' + mkdir layer + if [[ -n "$contents" ]]; then + echo "Adding contents..." + for item in $contents; do + echo "Adding $item" + rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/ + done + else + echo "No contents to add to layer." + fi + + chmod ug+w layer + + if [[ -n $extraCommands ]]; then + (cd layer; eval "$extraCommands") + fi + + # Tar up the layer and throw it into 'layer.tar'. + echo "Packing layer..." + mkdir $out + tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar . + + # Compute a checksum of the tarball. + echo "Computing layer checksum..." + tarsum=$(${tarsum} < $out/layer.tar) + + # Add a 'checksum' field to the JSON, with the value set to the + # checksum of the tarball. + cat ${baseJson} | jshon -s "$tarsum" -i checksum > $out/json + + # Indicate to docker that we're using schema version 1.0. + echo -n "1.0" > $out/VERSION + + echo "Finished building layer '${name}'" + ''; + + # Make a "root" layer; required if we need to execute commands as a + # privileged user on the image. The commands themselves will be + # performed in a virtual machine sandbox. + mkRootLayer = { + # Name of the image. + name, + # Script to run as root. Bash. + runAsRoot, + # Files to add to the layer. If null, an empty layer will be created. + contents ? null, + # When copying the contents into the image, preserve symlinks to + # directories (see `rsync -K`). Otherwise, transform those symlinks + # into directories. + keepContentsDirlinks ? false, + # JSON containing configuration and metadata for this layer. + baseJson, + # Existing image onto which to append the new layer. + fromImage ? null, + # Name of the image we're appending onto. + fromImageName ? null, + # Tag of the image we're appending onto. + fromImageTag ? null, + # How much disk to allocate for the temporary virtual machine. + diskSize ? 1024, + # Commands (bash) to run on the layer; these do not require sudo. + extraCommands ? "" + }: + # Generate an executable script from the `runAsRoot` text. + let runAsRootScript = shellScript "run-as-root.sh" runAsRoot; + in runWithOverlay { + name = "docker-layer-${name}"; + + inherit fromImage fromImageName fromImageTag diskSize; + + preMount = lib.optionalString (contents != null && contents != []) '' + echo "Adding contents..." + for item in ${toString contents}; do + echo "Adding $item..." + rsync -a${if keepContentsDirlinks then "K" else "k"} --chown=0:0 $item/ layer/ + done + + chmod ug+w layer + ''; + + postMount = '' + mkdir -p mnt/{dev,proc,sys} mnt${storeDir} + + # Mount /dev, /sys and the nix store as shared folders. + mount --rbind /dev mnt/dev + mount --rbind /sys mnt/sys + mount --rbind ${storeDir} mnt${storeDir} + + # Execute the run as root script. See 'man unshare' for + # details on what's going on here; basically this command + # means that the runAsRootScript will be executed in a nearly + # completely isolated environment. + unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript} + + # Unmount directories and remove them. + umount -R mnt/dev mnt/sys mnt${storeDir} + rmdir --ignore-fail-on-non-empty \ + mnt/dev mnt/proc mnt/sys mnt${storeDir} \ + mnt$(dirname ${storeDir}) + ''; + + postUmount = '' + (cd layer; eval "${extraCommands}") + + echo "Packing layer..." + mkdir $out + tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar . + + # Compute the tar checksum and add it to the output json. + echo "Computing checksum..." + ts=$(${tarsum} < $out/layer.tar) + cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json + # Indicate to docker that we're using schema version 1.0. + echo -n "1.0" > $out/VERSION + + echo "Finished building layer '${name}'" + ''; + }; + + # 1. extract the base image + # 2. create the layer + # 3. add layer deps to the layer itself, diffing with the base image + # 4. compute the layer id + # 5. put the layer in the image + # 6. repack the image + buildImage = args@{ + # Image name. + name, + # Image tag. + tag ? "latest", + # Parent image, to append to. + fromImage ? null, + # Name of the parent image; will be read from the image otherwise. + fromImageName ? null, + # Tag of the parent image; will be read from the image otherwise. + fromImageTag ? null, + # Files to put on the image (a nix store path or list of paths). + contents ? null, + # When copying the contents into the image, preserve symlinks to + # directories (see `rsync -K`). Otherwise, transform those symlinks + # into directories. + keepContentsDirlinks ? false, + # Docker config; e.g. what command to run on the container. + config ? null, + # Optional bash script to run on the files prior to fixturizing the layer. + extraCommands ? "", uid ? 0, gid ? 0, + # Optional bash script to run as root on the image when provisioning. + runAsRoot ? null, + # Size of the virtual machine disk to provision when building the image. + diskSize ? 1024, + # Time of creation of the image. + created ? "1970-01-01T00:00:01Z", + }: + + let + baseName = baseNameOf name; + + # Create a JSON blob of the configuration. Set the date to unix zero. + baseJson = writeText "${baseName}-config.json" (builtins.toJSON { + inherit created config; + architecture = "amd64"; + os = "linux"; + }); + + layer = + if runAsRoot == null + then mkPureLayer { + name = baseName; + inherit baseJson contents keepContentsDirlinks extraCommands uid gid; + } else mkRootLayer { + name = baseName; + inherit baseJson fromImage fromImageName fromImageTag + contents keepContentsDirlinks runAsRoot diskSize + extraCommands; + }; + result = runCommand "docker-image-${baseName}.tar.gz" { + buildInputs = [ jshon pigz coreutils findutils jq ]; + # Image name and tag must be lowercase + imageName = lib.toLower name; + imageTag = lib.toLower tag; + inherit fromImage baseJson; + layerClosure = writeReferencesToFile layer; + passthru.buildArgs = args; + passthru.layer = layer; + } '' + # Print tar contents: + # 1: Interpreted as relative to the root directory + # 2: With no trailing slashes on directories + # This is useful for ensuring that the output matches the + # values generated by the "find" command + ls_tar() { + for f in $(tar -tf $1 | xargs realpath -ms --relative-to=.); do + if [[ "$f" != "." ]]; then + echo "/$f" + fi + done + } + + mkdir image + touch baseFiles + if [[ -n "$fromImage" ]]; then + echo "Unpacking base image..." + tar -C image -xpf "$fromImage" + # Do not import the base image configuration and manifest + chmod a+w image image/*.json + rm -f image/*.json + + if [[ -z "$fromImageName" ]]; then + fromImageName=$(jshon -k < image/repositories|head -n1) + fi + if [[ -z "$fromImageTag" ]]; then + fromImageTag=$(jshon -e $fromImageName -k \ + < image/repositories|head -n1) + fi + parentID=$(jshon -e $fromImageName -e $fromImageTag -u \ + < image/repositories) + + for l in image/*/layer.tar; do + ls_tar $l >> baseFiles + done + fi + + chmod -R ug+rw image + + mkdir temp + cp ${layer}/* temp/ + chmod ug+w temp/* + + for dep in $(cat $layerClosure); do + find $dep >> layerFiles + done + + echo "Adding layer..." + # Record the contents of the tarball with ls_tar. + ls_tar temp/layer.tar >> baseFiles + + # Get the files in the new layer which were *not* present in + # the old layer, and record them as newFiles. + comm <(sort -n baseFiles|uniq) \ + <(sort -n layerFiles|uniq|grep -v ${layer}) -1 -3 > newFiles + # Append the new files to the layer. + tar -rpf temp/layer.tar --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 --no-recursion --files-from newFiles + + echo "Adding meta..." + + # If we have a parentID, add it to the json metadata. + if [[ -n "$parentID" ]]; then + cat temp/json | jshon -s "$parentID" -i parent > tmpjson + mv tmpjson temp/json + fi + + # Take the sha256 sum of the generated json and use it as the layer ID. + # Compute the size and add it to the json under the 'Size' field. + layerID=$(sha256sum temp/json|cut -d ' ' -f 1) + size=$(stat --printf="%s" temp/layer.tar) + cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson + mv tmpjson temp/json + + # Use the temp folder we've been working on to create a new image. + mv temp image/$layerID + + # Create image json and image manifest + imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}") + manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]") + currentID=$layerID + while [[ -n "$currentID" ]]; do + layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1) + imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"${created}\"}] + .") + imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .") + manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .") + + currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true)) + done + + imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1) + echo "$imageJson" > "image/$imageJsonChecksum.json" + manifestJson=$(echo "$manifestJson" | jq ".[0].Config = \"$imageJsonChecksum.json\"") + echo "$manifestJson" > image/manifest.json + + # Store the json under the name image/repositories. + jshon -n object \ + -n object -s "$layerID" -i "$imageTag" \ + -i "$imageName" > image/repositories + + # Make the image read-only. + chmod -R a-w image + + echo "Cooking the image..." + tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'./':: -c . | pigz -nT > $out + + echo "Finished." + ''; + + in + result; + + # Build an image and populate its nix database with the provided + # contents. The main purpose is to be able to use nix commands in + # the container. + # Be careful since this doesn't work well with multilayer. + buildImageWithNixDb = args@{ contents ? null, extraCommands ? "", ... }: + buildImage (args // { + extraCommands = '' + echo "Generating the nix database..." + echo "Warning: only the database of the deepest Nix layer is loaded." + echo " If you want to use nix commands in the container, it would" + echo " be better to only have one layer that contains a nix store." + # This requires Nix 1.12 or higher + export NIX_REMOTE=local?root=$PWD + ${nixUnstable}/bin/nix-store --load-db < ${nixRegistration contents}/db.dump + + # We fill the store in order to run the 'verify' command that + # generates hash and size of output paths. + # Note when Nix 1.12 is be the stable one, the database dump + # generated by the exportReferencesGraph function will + # contains sha and size. See + # https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a + storePaths=$(cat ${nixRegistration contents}/storePaths) + echo "Copying everything to /nix/store (will take a while)..." + cp -prd $storePaths nix/store/ + ${nixUnstable}/bin/nix-store --verify --check-contents + + mkdir -p nix/var/nix/gcroots/docker/ + for i in ${lib.concatStringsSep " " contents}; do + ln -s $i nix/var/nix/gcroots/docker/$(basename $i) + done; + '' + extraCommands; + }); +} diff --git a/nixos/overlays/detjson.py b/nixos/overlays/detjson.py new file mode 100644 index 0000000..439c213 --- /dev/null +++ b/nixos/overlays/detjson.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Deterministic layer json: https://github.com/docker/hub-feedback/issues/488 + +import sys +reload(sys) +sys.setdefaultencoding('UTF8') +import json + +# If any of the keys below are equal to a certain value +# then we can delete it because it's the default value +SAFEDELS = { + "Size": 0, + "config": { + "ExposedPorts": None, + "MacAddress": "", + "NetworkDisabled": False, + "PortSpecs": None, + "VolumeDriver": "" + } +} +SAFEDELS["container_config"] = SAFEDELS["config"] + +def makedet(j, safedels): + for k,v in safedels.items(): + if k not in j: + continue + if type(v) == dict: + makedet(j[k], v) + elif j[k] == v: + del j[k] + +def main(): + j = json.load(sys.stdin) + makedet(j, SAFEDELS) + json.dump(j, sys.stdout, sort_keys=True) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/nixos/overlays/examples.nix b/nixos/overlays/examples.nix new file mode 100644 index 0000000..3154403 --- /dev/null +++ b/nixos/overlays/examples.nix @@ -0,0 +1,127 @@ +# Examples of using the docker tools to build packages. +# +# This file defines several docker images. In order to use an image, +# build its derivation with `nix-build`, and then load the result with +# `docker load`. For example: +# +# $ nix-build '' -A dockerTools.examples.redis +# $ docker load < result + +{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }: + +rec { + # 1. basic example + bash = buildImage { + name = "bash"; + contents = pkgs.bashInteractive; + }; + + # 2. service example, layered on another image + redis = buildImage { + name = "redis"; + tag = "latest"; + + # for example's sake, we can layer redis on top of bash or debian + fromImage = bash; + # fromImage = debian; + + contents = pkgs.redis; + runAsRoot = '' + mkdir -p /data + ''; + + config = { + Cmd = [ "/bin/redis-server" ]; + WorkingDir = "/data"; + Volumes = { + "/data" = {}; + }; + }; + }; + + # 3. another service example + nginx = let + nginxPort = "80"; + nginxConf = pkgs.writeText "nginx.conf" '' + user nginx nginx; + daemon off; + error_log /dev/stdout info; + pid /dev/null; + events {} + http { + access_log /dev/stdout; + server { + listen ${nginxPort}; + index index.html; + location / { + root ${nginxWebRoot}; + } + } + } + ''; + nginxWebRoot = pkgs.writeTextDir "index.html" '' +

Hello from NGINX

+ ''; + in + buildImage { + name = "nginx-container"; + contents = pkgs.nginx; + + runAsRoot = '' + #!${pkgs.stdenv.shell} + ${shadowSetup} + groupadd --system nginx + useradd --system --gid nginx nginx + ''; + + config = { + Cmd = [ "nginx" "-c" nginxConf ]; + ExposedPorts = { + "${nginxPort}/tcp" = {}; + }; + }; + }; + + # 4. example of pulling an image. could be used as a base for other images + nixFromDockerHub = pullImage { + imageName = "nixos/nix"; + imageTag = "1.11"; + # this hash will need change if the tag is updated at docker hub + sha256 = "0nncn9pn5miygan51w34c2p9qssi96jgsaqv44dxxdprc8pg0g83"; + }; + + # 5. example of multiple contents, emacs and vi happily coexisting + editors = buildImage { + name = "editors"; + contents = [ + pkgs.coreutils + pkgs.bash + pkgs.emacs + pkgs.vim + pkgs.nano + ]; + }; + + # 6. nix example to play with the container nix store + # docker run -it --rm nix nix-store -qR $(nix-build '' -A nix) + nix = buildImageWithNixDb { + name = "nix"; + contents = [ + # nix-store uses cat program to display results as specified by + # the image env variable NIX_PAGER. + pkgs.coreutils + pkgs.nix + ]; + config = { + Env = [ "NIX_PAGER=cat" ]; + }; + }; + + # 7. example of adding something on top of an image pull by our + # dockerTools chain. + onTopOfPulledImage = buildImage { + name = "onTopOfPulledImage"; + fromImage = nixFromDockerHub; + contents = [ pkgs.hello ]; + }; +} diff --git a/nixos/overlays/kubernetes.nix b/nixos/overlays/kubernetes.nix new file mode 100644 index 0000000..53f4bcf --- /dev/null +++ b/nixos/overlays/kubernetes.nix @@ -0,0 +1,1157 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.kubernetes; + + skipAttrs = attrs: map (filterAttrs (k: v: k != "enable")) + (filter (v: !(hasAttr "enable" v) || v.enable) attrs); + + infraContainer = pkgs.dockerTools.buildImage { + name = "pause"; + tag = "latest"; + contents = cfg.package.pause; + config.Cmd = "/bin/pause"; + }; + + mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" ( + builtins.toJSON ( + let name' = + if name == "kubelet" + then "system:node:${config.services.kubernetes.kubelet.hostname}" + else if name == "kube-proxy" + then "system:kube-proxy" + else name; + in + { + apiVersion = "v1"; + kind = "Config"; + clusters = [{ + name = "local"; + cluster.certificate-authority = cfg.caFile; + cluster.server = cfg.server; + }]; + users = [{ + name = name'; + user = { + client-certificate = cfg.certFile; + client-key = cfg.keyFile; + }; + }]; + contexts = [{ + context = { + cluster = "local"; + user = name'; + }; + current-context = "default"; + }]; + })); + + mkKubeConfigOptions = prefix: { + server = mkOption { + description = "${prefix} kube-apiserver server address."; + default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}"; + type = types.str; + }; + + caFile = mkOption { + description = "${prefix} certificate authrority file used to connect to kube-apiserver."; + type = types.nullOr types.path; + default = cfg.caFile; + }; + + certFile = mkOption { + description = "${prefix} client certificate file used to connect to kube-apiserver."; + type = types.nullOr types.path; + default = null; + }; + + keyFile = mkOption { + description = "${prefix} client key file used to connect to kube-apiserver."; + type = types.nullOr types.path; + default = null; + }; + }; + + kubeConfigDefaults = { + server = mkDefault cfg.kubeconfig.server; + caFile = mkDefault cfg.kubeconfig.caFile; + certFile = mkDefault cfg.kubeconfig.certFile; + keyFile = mkDefault cfg.kubeconfig.keyFile; + }; + + cniConfig = pkgs.buildEnv { + name = "kubernetes-cni-config"; + paths = imap (i: entry: + pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) + ) cfg.kubelet.cni.config; + }; + + manifests = pkgs.buildEnv { + name = "kubernetes-manifests"; + paths = mapAttrsToList (name: manifest: + pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest) + ) cfg.kubelet.manifests; + }; + + addons = pkgs.runCommand "kubernetes-addons" { } '' + mkdir -p $out + # since we are mounting the addons to the addon manager, they need to be copied + ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon: + pkgs.writeTextDir "${name}.json" (builtins.toJSON addon) + ) (cfg.addonManager.addons))} + ''; + + taintOptions = { name, ... }: { + options = { + key = mkOption { + description = "Key of taint."; + default = name; + type = types.str; + }; + value = mkOption { + description = "Value of taint."; + type = types.str; + }; + effect = mkOption { + description = "Effect of taint."; + example = "NoSchedule"; + type = types.enum ["NoSchedule" "PreferNoSchedule" "NoExecute"]; + }; + }; + }; + + taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints); + + # needed for flannel to pass options to docker + mkDockerOpts = pkgs.runCommand "mk-docker-opts" { + buildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out + cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh + + # bashInteractive needed for `compgen` + makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh" + ''; +in { + + + ###### interface + + options.services.kubernetes = { + roles = mkOption { + description = '' + Kubernetes role that this machine should take. + + Master role will enable etcd, apiserver, scheduler and controller manager + services. Node role will enable etcd, docker, kubelet and proxy services. + ''; + default = []; + type = types.listOf (types.enum ["master" "node"]); + }; + + package = mkOption { + description = "Kubernetes package to use."; + type = types.package; + default = pkgs.kubernetes; + defaultText = "pkgs.kubernetes"; + }; + + verbose = mkOption { + description = "Kubernetes enable verbose mode for debugging."; + default = false; + type = types.bool; + }; + + etcd = { + servers = mkOption { + description = "List of etcd servers. By default etcd is started, except if this option is changed."; + default = ["http://127.0.0.1:2379"]; + type = types.listOf types.str; + }; + + keyFile = mkOption { + description = "Etcd key file."; + default = null; + type = types.nullOr types.path; + }; + + certFile = mkOption { + description = "Etcd cert file."; + default = null; + type = types.nullOr types.path; + }; + + caFile = mkOption { + description = "Etcd ca file."; + default = cfg.caFile; + type = types.nullOr types.path; + }; + }; + + kubeconfig = mkKubeConfigOptions "Default kubeconfig"; + + caFile = mkOption { + description = "Default kubernetes certificate authority"; + type = types.nullOr types.path; + default = null; + }; + + dataDir = mkOption { + description = "Kubernetes root directory for managing kubelet files."; + default = "/var/lib/kubernetes"; + type = types.path; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = []; + type = types.listOf types.str; + }; + + apiserver = { + enable = mkOption { + description = "Whether to enable Kubernetes apiserver."; + default = false; + type = types.bool; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = cfg.featureGates; + type = types.listOf types.str; + }; + + address = mkOption { + description = "Kubernetes apiserver listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + publicAddress = mkOption { + description = '' + Kubernetes apiserver public listening address used for read only and + secure port. + ''; + default = cfg.apiserver.address; + type = types.str; + }; + + advertiseAddress = mkOption { + description = '' + Kubernetes apiserver IP address on which to advertise the apiserver + to members of the cluster. This address must be reachable by the rest + of the cluster. + ''; + default = null; + type = types.nullOr types.str; + }; + + storageBackend = mkOption { + description = '' + Kubernetes apiserver storage backend. + ''; + default = "etcd3"; + type = types.enum ["etcd2" "etcd3"]; + }; + + port = mkOption { + description = "Kubernetes apiserver listening port."; + default = 8080; + type = types.int; + }; + + securePort = mkOption { + description = "Kubernetes apiserver secure port."; + default = 443; + type = types.int; + }; + + tlsCertFile = mkOption { + description = "Kubernetes apiserver certificate file."; + default = null; + type = types.nullOr types.path; + }; + + tlsKeyFile = mkOption { + description = "Kubernetes apiserver private key file."; + default = null; + type = types.nullOr types.path; + }; + + clientCaFile = mkOption { + description = "Kubernetes apiserver CA file for client auth."; + default = cfg.caFile; + type = types.nullOr types.path; + }; + + tokenAuthFile = mkOption { + description = '' + Kubernetes apiserver token authentication file. See + + ''; + default = null; + type = types.nullOr types.path; + }; + + basicAuthFile = mkOption { + description = '' + Kubernetes apiserver basic authentication file. See + + ''; + default = pkgs.writeText "users" '' + kubernetes,admin,0 + ''; + type = types.nullOr types.path; + }; + + authorizationMode = mkOption { + description = '' + Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See + + ''; + default = ["RBAC" "Node"]; + type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC" "Node"]); + }; + + authorizationPolicy = mkOption { + description = '' + Kubernetes apiserver authorization policy file. See + + ''; + default = []; + type = types.listOf types.attrs; + }; + + allowPrivileged = mkOption { + description = "Whether to allow privileged containers on Kubernetes."; + default = true; + type = types.bool; + }; + + serviceClusterIpRange = mkOption { + description = '' + A CIDR notation IP range from which to assign service cluster IPs. + This must not overlap with any IP ranges assigned to nodes for pods. + ''; + default = "10.0.0.0/24"; + type = types.str; + }; + + runtimeConfig = mkOption { + description = '' + Api runtime configuration. See + + ''; + default = "authentication.k8s.io/v1beta1=true"; + example = "api/all=false,api/v1=true"; + type = types.str; + }; + + admissionControl = mkOption { + description = '' + Kubernetes admission control plugins to use. See + + ''; + default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"]; + example = [ + "NamespaceLifecycle" "NamespaceExists" "LimitRanger" + "SecurityContextDeny" "ServiceAccount" "ResourceQuota" + "PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass" + ]; + type = types.listOf types.str; + }; + + serviceAccountKeyFile = mkOption { + description = '' + Kubernetes apiserver PEM-encoded x509 RSA private or public key file, + used to verify ServiceAccount tokens. By default tls private key file + is used. + ''; + default = null; + type = types.nullOr types.path; + }; + + kubeletClientCaFile = mkOption { + description = "Path to a cert file for connecting to kubelet."; + default = cfg.caFile; + type = types.nullOr types.path; + }; + + kubeletClientCertFile = mkOption { + description = "Client certificate to use for connections to kubelet."; + default = null; + type = types.nullOr types.path; + }; + + kubeletClientKeyFile = mkOption { + description = "Key to use for connections to kubelet."; + default = null; + type = types.nullOr types.path; + }; + + kubeletHttps = mkOption { + description = "Whether to use https for connections to kubelet."; + default = true; + type = types.bool; + }; + + extraOpts = mkOption { + description = "Kubernetes apiserver extra command line options."; + default = ""; + type = types.str; + }; + }; + + scheduler = { + enable = mkOption { + description = "Whether to enable Kubernetes scheduler."; + default = false; + type = types.bool; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = cfg.featureGates; + type = types.listOf types.str; + }; + + address = mkOption { + description = "Kubernetes scheduler listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + port = mkOption { + description = "Kubernetes scheduler listening port."; + default = 10251; + type = types.int; + }; + + leaderElect = mkOption { + description = "Whether to start leader election before executing main loop."; + type = types.bool; + default = true; + }; + + kubeconfig = mkKubeConfigOptions "Kubernetes scheduler"; + + extraOpts = mkOption { + description = "Kubernetes scheduler extra command line options."; + default = ""; + type = types.str; + }; + }; + + controllerManager = { + enable = mkOption { + description = "Whether to enable Kubernetes controller manager."; + default = false; + type = types.bool; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = cfg.featureGates; + type = types.listOf types.str; + }; + + address = mkOption { + description = "Kubernetes controller manager listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + port = mkOption { + description = "Kubernetes controller manager listening port."; + default = 10252; + type = types.int; + }; + + leaderElect = mkOption { + description = "Whether to start leader election before executing main loop."; + type = types.bool; + default = true; + }; + + serviceAccountKeyFile = mkOption { + description = '' + Kubernetes controller manager PEM-encoded private RSA key file used to + sign service account tokens + ''; + default = null; + type = types.nullOr types.path; + }; + + rootCaFile = mkOption { + description = '' + Kubernetes controller manager certificate authority file included in + service account's token secret. + ''; + default = cfg.caFile; + type = types.nullOr types.path; + }; + + kubeconfig = mkKubeConfigOptions "Kubernetes controller manager"; + + extraOpts = mkOption { + description = "Kubernetes controller manager extra command line options."; + default = ""; + type = types.str; + }; + }; + + kubelet = { + enable = mkOption { + description = "Whether to enable Kubernetes kubelet."; + default = false; + type = types.bool; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = cfg.featureGates; + type = types.listOf types.str; + }; + + seedDockerImages = mkOption { + description = "List of docker images to preload on system"; + default = []; + type = types.listOf types.package; + }; + + registerNode = mkOption { + description = "Whether to auto register kubelet with API server."; + default = true; + type = types.bool; + }; + + address = mkOption { + description = "Kubernetes kubelet info server listening address."; + default = "0.0.0.0"; + type = types.str; + }; + + port = mkOption { + description = "Kubernetes kubelet info server listening port."; + default = 10250; + type = types.int; + }; + + tlsCertFile = mkOption { + description = "File containing x509 Certificate for HTTPS."; + default = null; + type = types.nullOr types.path; + }; + + tlsKeyFile = mkOption { + description = "File containing x509 private key matching tlsCertFile."; + default = null; + type = types.nullOr types.path; + }; + + clientCaFile = mkOption { + description = "Kubernetes apiserver CA file for client authentication."; + default = cfg.caFile; + type = types.nullOr types.path; + }; + + healthz = { + bind = mkOption { + description = "Kubernetes kubelet healthz listening address."; + default = "127.0.0.1"; + type = types.str; + }; + + port = mkOption { + description = "Kubernetes kubelet healthz port."; + default = 10248; + type = types.int; + }; + }; + + hostname = mkOption { + description = "Kubernetes kubelet hostname override."; + default = config.networking.hostName; + type = types.str; + }; + + allowPrivileged = mkOption { + description = "Whether to allow Kubernetes containers to request privileged mode."; + default = true; + type = types.bool; + }; + + cadvisorPort = mkOption { + description = "Kubernetes kubelet local cadvisor port."; + default = 4194; + type = types.int; + }; + + clusterDns = mkOption { + description = "Use alternative DNS."; + default = "10.1.0.1"; + type = types.str; + }; + + clusterDomain = mkOption { + description = "Use alternative domain."; + default = config.services.kubernetes.addons.dns.clusterDomain; + type = types.str; + }; + + networkPlugin = mkOption { + description = "Network plugin to use by Kubernetes."; + type = types.nullOr (types.enum ["cni" "kubenet"]); + default = "kubenet"; + }; + + cni = { + packages = mkOption { + description = "List of network plugin packages to install."; + type = types.listOf types.package; + default = []; + }; + + config = mkOption { + description = "Kubernetes CNI configuration."; + type = types.listOf types.attrs; + default = []; + example = literalExample '' + [{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.22.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } + } { + "cniVersion": "0.2.0", + "type": "loopback" + }] + ''; + }; + }; + + manifests = mkOption { + description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; + type = types.attrsOf types.attrs; + default = {}; + }; + + applyManifests = mkOption { + description = "Whether to apply manifests (this is true for master node)."; + default = false; + type = types.bool; + }; + + unschedulable = mkOption { + description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; + default = false; + type = types.bool; + }; + + taints = mkOption { + description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)."; + default = {}; + type = types.attrsOf (types.submodule [ taintOptions ]); + }; + + nodeIp = mkOption { + description = "IP address of the node. If set, kubelet will use this IP address for the node."; + default = null; + type = types.nullOr types.str; + }; + + kubeconfig = mkKubeConfigOptions "Kubelet"; + + extraOpts = mkOption { + description = "Kubernetes kubelet extra command line options."; + default = ""; + type = types.str; + }; + }; + + proxy = { + enable = mkOption { + description = "Whether to enable Kubernetes proxy."; + default = false; + type = types.bool; + }; + + featureGates = mkOption { + description = "List set of feature gates"; + default = cfg.featureGates; + type = types.listOf types.str; + }; + + address = mkOption { + description = "Kubernetes proxy listening address."; + default = "0.0.0.0"; + type = types.str; + }; + + kubeconfig = mkKubeConfigOptions "Kubernetes proxy"; + + extraOpts = mkOption { + description = "Kubernetes proxy extra command line options."; + default = ""; + type = types.str; + }; + }; + + addonManager = { + enable = mkOption { + description = "Whether to enable Kubernetes addon manager."; + default = false; + type = types.bool; + }; + + addons = mkOption { + description = "Kubernetes addons (any kind of Kubernetes resource can be an addon)."; + default = { }; + type = types.attrsOf (types.either types.attrs (types.listOf types.attrs)); + example = literalExample '' + { + "my-service" = { + "apiVersion" = "v1"; + "kind" = "Service"; + "metadata" = { + "name" = "my-service"; + "namespace" = "default"; + }; + "spec" = { ... }; + }; + } + // import { cfg = config.services.kubernetes; }; + ''; + }; + }; + + path = mkOption { + description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added."; + type = types.listOf types.package; + default = []; + }; + + clusterCidr = mkOption { + description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster."; + default = "10.1.0.0/16"; + type = types.str; + }; + + flannel.enable = mkOption { + description = "Whether to enable flannel networking"; + default = false; + type = types.bool; + }; + + }; + + ###### implementation + + config = mkMerge [ + (mkIf cfg.kubelet.enable { + services.kubernetes.kubelet.seedDockerImages = [infraContainer]; + + systemd.services.kubelet-bootstrap = { + description = "Boostrap Kubelet"; + wantedBy = ["kubernetes.target"]; + after = ["docker.service" "network.target"]; + path = with pkgs; [ docker ]; + script = '' + ${concatMapStrings (img: '' + echo "Seeding docker image: ${img}" + docker load <${img} + '') cfg.kubelet.seedDockerImages} + + rm /opt/cni/bin/* || true + ${concatMapStrings (package: '' + echo "Linking cni package: ${package}" + ln -fs ${package}/bin/* /opt/cni/bin + '') cfg.kubelet.cni.packages} + ''; + serviceConfig = { + Slice = "kubernetes.slice"; + Type = "oneshot"; + }; + }; + + systemd.services.kubelet = { + description = "Kubernetes Kubelet Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ]; + path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${cfg.package}/bin/kubelet \ + ${optionalString cfg.kubelet.applyManifests + "--pod-manifest-path=${manifests}"} \ + ${optionalString (taints != "") + "--register-with-taints=${taints}"} \ + --kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \ + --require-kubeconfig \ + --address=${cfg.kubelet.address} \ + --port=${toString cfg.kubelet.port} \ + --register-node=${boolToString cfg.kubelet.registerNode} \ + ${optionalString (cfg.kubelet.tlsCertFile != null) + "--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \ + ${optionalString (cfg.kubelet.tlsKeyFile != null) + "--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \ + ${optionalString (cfg.kubelet.clientCaFile != null) + "--client-ca-file=${cfg.kubelet.clientCaFile}"} \ + --authentication-token-webhook \ + --authentication-token-webhook-cache-ttl="10s" \ + --authorization-mode=Webhook \ + --healthz-bind-address=${cfg.kubelet.healthz.bind} \ + --healthz-port=${toString cfg.kubelet.healthz.port} \ + --hostname-override=${cfg.kubelet.hostname} \ + --allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \ + --root-dir=${cfg.dataDir} \ + --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \ + ${optionalString (cfg.kubelet.clusterDns != "") + "--cluster-dns=${cfg.kubelet.clusterDns}"} \ + ${optionalString (cfg.kubelet.clusterDomain != "") + "--cluster-domain=${cfg.kubelet.clusterDomain}"} \ + --pod-infra-container-image=pause \ + ${optionalString (cfg.kubelet.networkPlugin != null) + "--network-plugin=${cfg.kubelet.networkPlugin}"} \ + --cni-conf-dir=${cniConfig} \ + --hairpin-mode=hairpin-veth \ + ${optionalString (cfg.kubelet.nodeIp != null) + "--node-ip=${cfg.kubelet.nodeIp}"} \ + ${optionalString (cfg.kubelet.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \ + ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \ + ${cfg.kubelet.extraOpts} + ''; + WorkingDirectory = cfg.dataDir; + }; + }; + + # Allways include cni plugins + services.kubernetes.kubelet.cni.packages = [pkgs.cni]; + + boot.kernelModules = ["br_netfilter"]; + + services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults; + }) + + (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) { + environment.etc = mapAttrs' (name: manifest: + nameValuePair "kubernetes/manifests/${name}.json" { + text = builtins.toJSON manifest; + mode = "0755"; + } + ) cfg.kubelet.manifests; + }) + + (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) { + services.kubernetes.kubelet.taints.unschedulable = { + value = "true"; + effect = "NoSchedule"; + }; + }) + + (mkIf cfg.apiserver.enable { + systemd.services.kube-apiserver = { + description = "Kubernetes Kubelet Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "network.target" "docker.service" ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${cfg.package}/bin/kube-apiserver \ + --etcd-servers=${concatStringsSep "," cfg.etcd.servers} \ + ${optionalString (cfg.etcd.caFile != null) + "--etcd-cafile=${cfg.etcd.caFile}"} \ + ${optionalString (cfg.etcd.certFile != null) + "--etcd-certfile=${cfg.etcd.certFile}"} \ + ${optionalString (cfg.etcd.keyFile != null) + "--etcd-keyfile=${cfg.etcd.keyFile}"} \ + --insecure-port=${toString cfg.apiserver.port} \ + --bind-address=0.0.0.0 \ + ${optionalString (cfg.apiserver.advertiseAddress != null) + "--advertise-address=${cfg.apiserver.advertiseAddress}"} \ + --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\ + ${optionalString (cfg.apiserver.tlsCertFile != null) + "--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \ + ${optionalString (cfg.apiserver.tlsKeyFile != null) + "--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \ + ${optionalString (cfg.apiserver.tokenAuthFile != null) + "--token-auth-file=${cfg.apiserver.tokenAuthFile}"} \ + ${optionalString (cfg.apiserver.basicAuthFile != null) + "--basic-auth-file=${cfg.apiserver.basicAuthFile}"} \ + --kubelet-https=${if cfg.apiserver.kubeletHttps then "true" else "false"} \ + ${optionalString (cfg.apiserver.kubeletClientCaFile != null) + "--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \ + ${optionalString (cfg.apiserver.kubeletClientCertFile != null) + "--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \ + ${optionalString (cfg.apiserver.kubeletClientKeyFile != null) + "--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \ + ${optionalString (cfg.apiserver.clientCaFile != null) + "--client-ca-file=${cfg.apiserver.clientCaFile}"} \ + --authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \ + ${optionalString (elem "ABAC" cfg.apiserver.authorizationMode) + "--authorization-policy-file=${ + pkgs.writeText "kube-auth-policy.jsonl" + (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy) + }" + } \ + --secure-port=${toString cfg.apiserver.securePort} \ + --service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \ + ${optionalString (cfg.apiserver.runtimeConfig != "") + "--runtime-config=${cfg.apiserver.runtimeConfig}"} \ + --admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \ + ${optionalString (cfg.apiserver.serviceAccountKeyFile!=null) + "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \ + ${optionalString cfg.verbose "--v=6"} \ + ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ + --storage-backend=${cfg.apiserver.storageBackend} \ + ${optionalString (cfg.kubelet.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \ + ${cfg.apiserver.extraOpts} + ''; + WorkingDirectory = cfg.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + AmbientCapabilities = "cap_net_bind_service"; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + }) + + (mkIf cfg.scheduler.enable { + systemd.services.kube-scheduler = { + description = "Kubernetes Scheduler Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${cfg.package}/bin/kube-scheduler \ + --address=${cfg.scheduler.address} \ + --port=${toString cfg.scheduler.port} \ + --leader-elect=${boolToString cfg.scheduler.leaderElect} \ + --kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \ + ${optionalString cfg.verbose "--v=6"} \ + ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ + ${optionalString (cfg.scheduler.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.scheduler.featureGates}"} \ + ${cfg.scheduler.extraOpts} + ''; + WorkingDirectory = cfg.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + }; + }; + + services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults; + }) + + (mkIf cfg.controllerManager.enable { + systemd.services.kube-controller-manager = { + description = "Kubernetes Controller Manager Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + serviceConfig = { + RestartSec = "30s"; + Restart = "on-failure"; + Slice = "kubernetes.slice"; + ExecStart = ''${cfg.package}/bin/kube-controller-manager \ + --address=${cfg.controllerManager.address} \ + --port=${toString cfg.controllerManager.port} \ + --kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \ + --leader-elect=${boolToString cfg.controllerManager.leaderElect} \ + ${if (cfg.controllerManager.serviceAccountKeyFile!=null) + then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}" + else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \ + ${if (cfg.controllerManager.rootCaFile!=null) + then "--root-ca-file=${cfg.controllerManager.rootCaFile}" + else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \ + ${optionalString (cfg.clusterCidr!=null) + "--cluster-cidr=${cfg.clusterCidr}"} \ + --allocate-node-cidrs=true \ + ${optionalString (cfg.controllerManager.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.controllerManager.featureGates}"} \ + ${optionalString cfg.verbose "--v=6"} \ + ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ + ${cfg.controllerManager.extraOpts} + ''; + WorkingDirectory = cfg.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + }; + path = cfg.path; + }; + + services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults; + }) + + (mkIf cfg.proxy.enable { + systemd.services.kube-proxy = { + description = "Kubernetes Proxy Service"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod]; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = ''${cfg.package}/bin/kube-proxy \ + --kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \ + --bind-address=${cfg.proxy.address} \ + ${optionalString (cfg.proxy.featureGates != []) + "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.proxy.featureGates}"} \ + ${optionalString cfg.verbose "--v=6"} \ + ${optionalString cfg.verbose "--log-flush-frequency=1s"} \ + ${optionalString (cfg.clusterCidr!=null) + "--cluster-cidr=${cfg.clusterCidr}"} \ + ${cfg.proxy.extraOpts} + ''; + WorkingDirectory = cfg.dataDir; + }; + }; + + # kube-proxy needs iptables + networking.firewall.enable = mkDefault true; + + services.kubernetes.proxy.kubeconfig = kubeConfigDefaults; + }) + + (mkIf (any (el: el == "master") cfg.roles) { + virtualisation.docker.enable = mkDefault true; + services.kubernetes.kubelet.enable = mkDefault true; + services.kubernetes.kubelet.allowPrivileged = mkDefault true; + services.kubernetes.kubelet.applyManifests = mkDefault true; + services.kubernetes.apiserver.enable = mkDefault true; + services.kubernetes.scheduler.enable = mkDefault true; + services.kubernetes.controllerManager.enable = mkDefault true; + services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]); + services.kubernetes.addonManager.enable = mkDefault true; + services.kubernetes.proxy.enable = mkDefault true; + }) + + # if this node is only a master make it unschedulable by default + (mkIf (all (el: el == "master") cfg.roles) { + services.kubernetes.kubelet.unschedulable = mkDefault true; + }) + + (mkIf (any (el: el == "node") cfg.roles) { + virtualisation.docker = { + enable = mkDefault true; + + # kubernetes needs access to logs + logDriver = mkDefault "json-file"; + + # iptables must be disabled for kubernetes + extraOptions = "--iptables=false --ip-masq=false"; + }; + + services.kubernetes.kubelet.enable = mkDefault true; + services.kubernetes.proxy.enable = mkDefault true; + }) + + (mkIf cfg.addonManager.enable { + environment.etc."kubernetes/addons".source = "${addons}/"; + + systemd.services.kube-addon-manager = { + description = "Kubernetes addon manager"; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + environment.ADDON_PATH = "/etc/kubernetes/addons/"; + serviceConfig = { + Slice = "kubernetes.slice"; + ExecStart = "${cfg.package}/bin/kube-addons"; + WorkingDirectory = cfg.dataDir; + User = "kubernetes"; + Group = "kubernetes"; + }; + }; + }) + + (mkIf ( + cfg.apiserver.enable || + cfg.scheduler.enable || + cfg.controllerManager.enable || + cfg.kubelet.enable || + cfg.proxy.enable + ) { + systemd.targets.kubernetes = { + description = "Kubernetes"; + wantedBy = [ "multi-user.target" ]; + }; + + systemd.tmpfiles.rules = [ + "d /opt/cni/bin 0755 root root -" + "d /var/run/kubernetes 0755 kubernetes kubernetes -" + "d /var/lib/kubernetes 0755 kubernetes kubernetes -" + ]; + + environment.systemPackages = [ cfg.package ]; + users.extraUsers = singleton { + name = "kubernetes"; + uid = config.ids.uids.kubernetes; + description = "Kubernetes user"; + extraGroups = [ "docker" ]; + group = "kubernetes"; + home = cfg.dataDir; + createHome = true; + }; + users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes; + + # dns addon is enabled by default + services.kubernetes.addons.dns.enable = mkDefault true; + }) + + (mkIf cfg.flannel.enable { + services.flannel = { + enable = mkDefault true; + network = mkDefault cfg.clusterCidr; + etcd = mkDefault { + endpoints = cfg.etcd.servers; + inherit (cfg.etcd) caFile certFile keyFile; + }; + }; + + services.kubernetes.kubelet = { + networkPlugin = mkDefault "cni"; + cni.config = mkDefault [{ + name = "mynet"; + type = "flannel"; + delegate = { + isDefaultGateway = true; + bridge = "docker0"; + }; + }]; + }; + + systemd.services."mk-docker-opts" = { + description = "Pre-Docker Actions"; + wantedBy = [ "flannel.service" ]; + before = [ "docker.service" ]; + after = [ "flannel.service" ]; + path = [ pkgs.gawk pkgs.gnugrep ]; + script = '' + mkdir -p /run/flannel + ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker + ''; + serviceConfig.Type = "oneshot"; + }; + systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; + + # read environment variables generated by mk-docker-opts + virtualisation.docker.extraOptions = "$DOCKER_OPTS"; + + networking.firewall.allowedUDPPorts = [ + 8285 # flannel udp + 8472 # flannel vxlan + ]; + }) + ]; +} diff --git a/nixos/overlays/overlays.nix b/nixos/overlays/overlays.nix new file mode 100644 index 0000000..ca3fa2b --- /dev/null +++ b/nixos/overlays/overlays.nix @@ -0,0 +1,4 @@ +self: super: { + dockerTools = super.callPackage ./overlay/default.nix { go = self.go_1_9; }; + # super.config.services.kubernetes = super.callPackage ./overlay/kubernetes.nix {}; +} diff --git a/nixos/overlays/pull.nix b/nixos/overlays/pull.nix new file mode 100644 index 0000000..5611c77 --- /dev/null +++ b/nixos/overlays/pull.nix @@ -0,0 +1,32 @@ +{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }: +let + nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name; +in +# For simplicity we only support sha256. +{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}" +, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }: +let + pullImage = vmTools.runInLinuxVM ( + stdenv.mkDerivation { + inherit name imageId; + + certs = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + + builder = ./pull.sh; + + nativeBuildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ]; + + outputHashAlgo = "sha256"; + outputHash = sha256; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + + preVM = vmTools.createEmptyImage { + size = 2048; + fullName = "${name}-disk"; + }; + + QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0"; + }); +in + pullImage diff --git a/nixos/overlays/pull.sh b/nixos/overlays/pull.sh new file mode 100644 index 0000000..0b1e9f3 --- /dev/null +++ b/nixos/overlays/pull.sh @@ -0,0 +1,36 @@ +source $stdenv/setup + +mkdir -p /var/lib/docker +mkfs.ext4 /dev/vda +mount -t ext4 /dev/vda /var/lib/docker + +modprobe virtio_net +dhclient eth0 + +mkdir -p /etc/ssl/certs/ +cp "$certs" "/etc/ssl/certs/" + +# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount +mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup +cd /sys/fs/cgroup +for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do + mkdir -p $sys + if ! mountpoint -q $sys; then + if ! mount -n -t cgroup -o $sys cgroup $sys; then + rmdir $sys || true + fi + fi +done + +# run docker daemon +dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock & + +until docker ps 2>/dev/null; do + printf '.' + sleep 1 +done + +rm -r $out + +docker pull ${imageId} +docker save ${imageId} > $out diff --git a/nixos/overlays/tarsum.go b/nixos/overlays/tarsum.go new file mode 100644 index 0000000..f91a90b --- /dev/null +++ b/nixos/overlays/tarsum.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "github.com/docker/docker/pkg/tarsum" +) + +func main() { + ts, err := tarsum.NewTarSum(os.Stdin, true, tarsum.Version1) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if _, err = io.Copy(ioutil.Discard, ts); err != nil { + fmt.Println(err) + os.Exit(1) + } + + fmt.Println(ts.Sum(nil)) +} diff --git a/base/packages.nix b/nixos/packages.nix similarity index 100% rename from base/packages.nix rename to nixos/packages.nix diff --git a/base/users.nix b/nixos/users.nix similarity index 100% rename from base/users.nix rename to nixos/users.nix