Major revamp and restructuring of k8s setup:

* replicated etcd setup for redundancy
* use Flannel network overlay
* naming conventions
* new repo structure
* etc.

base/configuration.nix

@@ -0,0 +1,40 @@
+{ config, pkgs, ... }:
+{
+  # Use the GRUB 2 boot loader.
+  boot.loader.grub.enable = true;
+  boot.loader.grub.version = 2;
+  boot.loader.grub.device = "/dev/sda";
+  boot.kernel.sysctl."vm.overcommit_memory"= 1;
+
+  services.vmwareGuest.enable = true;
+
+  # Select internationalisation properties.
+  i18n = {
+     consoleFont = "Lat2-Terminus16";
+     consoleKeyMap = "us";
+     defaultLocale = "en_US.UTF-8";
+  };
+
+  # Set your time zone.
+  time.timeZone = "Europe/Oslo";
+
+  networking.search = [ "itpartner.intern" "itpartner.no" ];
+  # Enable the OpenSSH daemon.
+  services.openssh.enable = true;
+
+  # The NixOS release to be compatible with for stateful data such as databases.
+  system.stateVersion = "17.03";
+  programs.zsh.enable = true;
+  programs.tmux.enable = true;
+
+  virtualisation.docker.enable = true;
+
+  security.rtkit.enable = true;
+
+  fileSystems."/data" = {
+     device = "10.253.18.103:/data";
+     fsType = "nfs";
+   };
+
+  imports = [ ./users.nix ./packages.nix ];
+}

base/packages.nix

@@ -0,0 +1,64 @@
+{ config, pkgs, ... }:
+let
+  nix-home = pkgs.callPackage ./nix-home.nix {};
+in
+  {
+    nixpkgs.config.allowUnfree = true;
+
+    environment.systemPackages =
+      with pkgs;
+      let
+        sys = [
+          stdenv
+          findutils
+          coreutils
+          psmisc
+          iputils
+          nettools
+          netcat
+          rsync
+          htop
+          iotop
+          zsh
+          wget
+          vimNox
+          python
+          file
+          bc
+          sshuttle
+          nix-prefetch-git
+          docker
+          nix-home
+        ];
+        devel = [
+          git
+          patchelf
+          binutils
+          gcc
+        ];
+        dotnet = with dotnetPackages; [
+          fsharp
+          mono-addins
+          mono
+          Fake
+          Paket
+        ];
+        node = with nodePackages; [
+          nodejs
+          npm
+          npm2nix
+          yo
+          gulp
+        ];
+        k8s = [
+          ebtables
+          ethtool
+          socat
+        ];
+      in
+      devel ++
+      dotnet ++
+      node ++
+      k8s ++
+      sys;
+}

base/users.nix

@@ -26,4 +26,17 @@
     "ssh-dss AAAAB3NzaC1kc3MAAACBANmiPELldjlhW4SKi9NVVN8DIpRouRj2j/v0ycySHYJv0lCE2ATggXMWY/T25eqMTEtwT7U+7g0MIHxR+GLFLpp6N7CiKh3pS0nj4Ig+f9hX2PF5HR6fOgQVCWcNrQTOV6jjZqWjbZpKYDpVfwHZxWrhgAv+9I+w0MHICulDca5RAAAAFQCjmmSZGG137bgGuLPXZkyvc8DXswAAAIB1bMTmPWS1qsZ5H4hgzoKcW+5b+yD7Yn62GFmZS/n4RdyJt7gBJwxukXaTs9B5g922lem4Tk6W6kslCzAu6Y7JDOkhX/hWasb6fGdCmmK/btqwi2imGeVJImAoFoTKfm4JprKcOmSATGMgTlzFHYFDpngyZ9pFnyubI829zfzNTwAAAIEAhwQwqEhBOT8cEKZRiDExi7jBk7zRKYhX1Wb6uUKI07qQFTLehUIahirHxqXcDhlcxzgHcwXKt6CBPYvre9qhqP6865Be/KecYycntVsx/o77Hv5bqETXojhLhb8I3hD1UnxIJ1FXVOnhL+SbO46oICsghBApDtgTX9iRFnuw0fU= jonas"
     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvIIQoi0aM2XmX7evnte/sQTCczSYYg0O0KT7g6Xao4nAoiMZ7udxOijd0vD8VBRSVuz6epcuPLsD0z6skiCJFpT0gG0KkTS6dLZMD3+KOvmvolpuGvRv6Ad/bO05YUvGHJAPdwwCxcXajtBoHOd+KUq8xBqyexgi20i+4P/JulY+RQKPlnQHlb6glcDAjt9RPh96t9T5lCoMAqMWtzV9GZE8/H+o6nMf9pxTjxT/oW/8EKZVgDgCSnpZg668Xj0UNcJW/ba3kSpjUsrdvZgM1E3TVgJ/YZDpM01m9hHS7PBcsJu6RyUMhamAVlsYS4Vy2ylU5rYAPTh23CZh24KD3+AtzTd9vhDNLz+KNKDzgW5b3IHoOyXG78RtPw7gHBGlGQc9OlJqTSLZAFWE2PNP1Pa6q0mqWcJuakSwyQoHvS+8PgEBr0eKKebwuXauuft1DvMNpi9SFxJY5Oy7ck62WzbBDNFDOTdNabXUV/QdOv5Zc2XrUCkr6C1dk57C0G8KTJCkqZIV1XMYZIlDZPuqlK8jZdPjRAnRmkYvvitb0FsvixpsWll1PRj9bYYZ01xX6djuDRvc1FeRG199HKOVWN53l9EMTpy0coxl2hZpbnyfxLdS/tkPkoSuWeY/MKWrn4yd1Q2+edjhTvnn6o6kfnKXKBkuoQvdaX/nyig+X+Q== august.s.solvang@gmail.com"
   ];
+
+  security.sudo.wheelNeedsPassword = false;
+  security.sudo.extraConfig =
+    ''
+      Defaults:root,%wheel env_keep+=LOCALE_ARCHIVE
+      Defaults:root,%wheel env_keep+=NIX_PATH
+      Defaults:root,%wheel env_keep+=TERMINFO_DIRS
+      Defaults env_keep+=SSH_AUTH_SOCK
+      Defaults lecture=never
+      Defaults shell_noargs
+      root   ALL=(ALL) SETENV: ALL
+      %wheel ALL=(ALL) NOPASSWD: ALL, SETENV: ALL
+     '';
 }

git.nix

@@ -1,7 +1,19 @@
 {
   git01 = { config, lib, pkgs, ... }:
-    {
+  {
-      deployment.targetHost = "10.253.18.103";
+    deployment.targetHost = "10.253.18.103";
-      imports = [ ./git01/configuration.nix ];
+    networking.hostName = "git01"; # Define your hostname
+    imports = [ ./hw/git01.nix ./git01/configuration.nix ];
+    services.nfs.server = {
+      enable=true;
+      exports= ''
+        /data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
+        /data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
+        /data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
+        /data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
+      '';
     };
+    networking.firewall.allowedTCPPorts = [2049 111 20048];
+    networking.firewall.allowedUDPPorts = [2049 111 20048];
+  };
 }

git01/configuration.nix

@@ -1,24 +0,0 @@
-{ config, pkgs, ... }:
-{
-  imports = [
-    ./hardware-configuration.nix
-    ../share/configuration.nix
-    ../share/users.nix
-  ];
-
-  networking.hostName = "git01"; # Define your hostname
-  networking.firewall.allowedTCPPorts = [2049 111 20048];
-  networking.firewall.allowedUDPPorts = [2049 111 20048];
-  boot.loader.grub.device = "/dev/sda";
-
-  services.nfs.server = {
-    enable=true;
-
-    exports= ''
-      /data         10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-      /data         10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-      /data         10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-      /data         10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
-    '';
-  };
-}

k8s.nix

@@ -1,19 +1,129 @@
-{
+let
-  k8s00 = { config, lib, pkgs, ... }:
+  etcdConfig = name: {
-    {
+    services.etcd = {
-      deployment.targetHost = "10.253.18.101";
+      inherit name;
-      imports = [ ./k8s00/configuration.nix ];
+      advertiseClientUrls = [ "https://${name}:2379" ];
+      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
+      enable = true;
+      certFile = ./pki/etcd.pem;
+      keyFile = ./pki/etcd-key.pem;
+      trustedCaFile = ./pki/ca.pem;
+      peerClientCertAuth = true;
+      listenClientUrls = ["https://0.0.0.0:2379"];
+      listenPeerUrls = ["https://0.0.0.0:2380"];
+      initialCluster = [
+        "etcd0=https://etcd0:2380"
+        "etcd1=https://etcd1:2380"
+      ];
+      # environment.variables = {
+        #   ETCDCTL_CERT_FILE = ./pki/client.pem;
+        #   ETCDCTL_KEY_FILE = ./pki/client-key.pem;
+        #   ETCDCTL_CA_FILE = ./pki/ca.pem;
+        #   ETCDCTL_PEERS = "https://127.0.0.1:2379";
+        # };
+    };
+    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+  };
+
+  flannelConfig = {
+    services.flannel = {
+      enable = true;
+      network = "10.10.0.0/16";
+      iface = "enp0s3";
+      etcd = {
+        endpoints = ["https://etcd0:2379" "https://etcd1:2379" ];
+        certFile = ./pki/client.pem;
+        keyFile = ./pki/client-key.pem;
+        caFile = ./pki/ca.pem;
+      };
+    };
+  };
+
+  kubeNode = {
+    services.kubernetes = {
+      # verbose = true;
+      roles = [ "node" ];
+      kubeconfig = {
+        server = "https://kubernetes:443";
+        caFile = ./pki/ca.pem;
+        certFile = ./pki/client.pem;
+        keyFile = ./pki/client-key.pem;
+      };
+      etcd = {
+        servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
+        certFile = ./pki/client.pem;
+        keyFile = ./pki/client-key.pem;
+        caFile = ./pki/ca.pem;
+      };
+      # kubelet.clusterDns = "10.10.1.1";
     };
 
-  k8s01 = { config, lib, pkgs, ... }:
+    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
+    networking.firewall.allowedTCPPorts = [ 10250 ];
+    networking.extraHosts = ''
+          10.253.18.100 etcd0 kubernetes
+          10.253.18.101 etcd1
+    '';
+
+    systemd.services.docker.after = [ "flannel.service" ];
+    systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
+    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
+  };
+
+  kubeMaster = {
+    services.dockerRegistry = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+    };
+
+    services.kubernetes = {
+      roles = [ "master" ];
+      apiserver = {
+        publicAddress = "0.0.0.0";
+        address = "0.0.0.0";
+        clientCaFile = ./pki/ca.pem;
+        tlsCertFile = ./pki/apiserver.pem;
+        tlsKeyFile = ./pki/apiserver-key.pem;
+        kubeletClientCaFile = ./pki/ca.pem;
+        kubeletClientCertFile = ./pki/client.pem;
+        kubeletClientKeyFile = ./pki/client-key.pem;
+      };
+      scheduler.leaderElect = true;
+      controllerManager.leaderElect = true;
+      controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
+    };
+
+    networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
+    networking.firewall.allowedUDPPorts = [ 53 ];
+    systemd.services.flannel.after = [ "etcd.service" ];
+  };
+in
+{
+  k8s0-0 = { config, lib, pkgs, ... }:
+    let etcd = etcdConfig "etcd0"; in
     {
       deployment.targetHost = "10.253.18.100";
-      imports = [ ./k8s01/configuration.nix ];
+      networking.hostName = "k8s0-0";
+      imports = [ ./hw/k8s0-0.nix ./base/configuration.nix ];
+      require = [ etcd flannelConfig ];
+      # require = [ etcd flannelConfig kubeMaster kubeNode ];
     };
 
-  k8s02 = { config, lib, pkgs, ... }:
+  k8s0-1 = { config, lib, pkgs, ... }:
+    let etcd = etcdConfig "etcd1"; in
+    {
+      deployment.targetHost = "10.253.18.101";
+      networking.hostName = "k8s0-1";
+      imports = [ ./hw/k8s0-1.nix ./base/configuration.nix ];
+      require = [ etcd flannelConfig  ];
+      # require = [ etcd flannelConfig kubeNode ];
+  };
+
+  k8s0-2 = { config, lib, pkgs, ... }:
     {
       deployment.targetHost = "10.253.18.102";
-      imports = [ ./k8s02/configuration.nix ];
+      networking.hostName = "k8s0-2";
-    };
+      imports = [ ./hw/k8s0-2.nix ./base/configuration.nix ];
+      require = [ flannelConfig  ];
+  };
 }

k8s0-0/configuration.nix

@@ -1,35 +0,0 @@
-{ config, pkgs, ... }:
-{
-  imports = [
-    ./hardware-configuration.nix
-    ../share/configuration.nix
-    ../share/users.nix
-    ../share/k8s.nix
-  ];
-
-  networking.hostName = "kubemaster"; # Define your hostname.
-  networking.firewall.allowedTCPPorts = [ 5000 8080 443 ];
-
-  boot.loader.grub.device = "/dev/sda";
-
-  services.dockerRegistry = {
-    enable = true;
-    listenAddress = "0.0.0.0";
-  };
-
-  services.kubernetes.roles = [ "master" ];
-  services.kubernetes.apiserver = {
-    publicAddress = "0.0.0.0";
-    address = "0.0.0.0";
-    # kubeletClientCaFile = ../pki/ca.pem;
-    # kubeletClientCertFile = ../pki/client.pem;
-    # kubeletClientKeyFile = ../pki/client-key.pem;
-    clientCaFile = ../pki/ca.pem;
-    tlsCertFile = ../pki/apiserver.pem;
-    tlsKeyFile = ../pki/apiserver-key.pem;
-  };
-  services.kubernetes.kubeconfig = {
-    certFile = ../pki/kubemaster.pem;
-    keyFile = ../pki/kubemaster-key.pem;
-  };
-}

k8s0-1/configuration.nix

@@ -1,18 +0,0 @@
-{ config, pkgs, ... }:
-{
-  imports = [
-    ./hardware-configuration.nix
-    ../share/configuration.nix
-    ../share/users.nix
-    ../share/k8s.nix
-  ];
-
-  boot.loader.grub.device = "/dev/sda";
-
-  networking.hostName = "kub01"; # Define your hostname.
-
-  services.kubernetes.kubeconfig = {
-    certFile = ../pki/kub01.pem;
-    keyFile = ../pki/kub01-key.pem;
-  };
-}

k8s0-2/configuration.nix

@@ -1,17 +0,0 @@
-{ config, pkgs, ... }:
-{
-  imports = [
-    ./hardware-configuration.nix
-    ../share/configuration.nix
-    ../share/users.nix
-    ../share/k8s.nix
-  ];
-  boot.loader.grub.device = "/dev/sda";
-
-  networking.hostName = "kub02"; # Define your hostname.
-
-  services.kubernetes.kubeconfig = {
-    certFile = ../pki/kub02.pem;
-    keyFile = ../pki/kub02-key.pem;
-  };
-}

pki/apiserver.json

@@ -1,7 +1,7 @@
 {
     "hosts": [
-        "kubemaster",
+        "k8s0-0",
-        "10.253.18.101"
+        "10.253.18.100"
     ],
     "key": {
         "algo": "rsa",

pki/ca.json

@@ -1,5 +1,6 @@
 {
     "hosts": [
+        "itpartner.no",
         "itpartner.intern",
         "cluster.local"
     ],

pki/client.json

@@ -1,9 +1,4 @@
 {
-    "CN": "@host@",
-    "hosts": [
-        "@host@",
-        "@ip@"
-    ],
     "key": {
         "algo": "rsa",
         "size": 2048

pki/etcd.json

@@ -0,0 +1,19 @@
+{
+    "hosts": [
+        "etcd0",
+        "etcd1"
+    ],
+    "key": {
+        "algo": "rsa",
+        "size": 2048
+    },
+    "names": [
+        {
+            "C": "NO",
+            "L": "Tromsø",
+            "O": "Serit IT Partner Tromsø AS",
+            "OU": "",
+            "ST": ""
+        }
+    ]
+}

pki/mkcerts.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-hosts="kubemaster,101 kub01,100 kub02,102"
+# hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102"
 
 mkcacert () {
     cfssl genkey -initca ca.json | cfssljson -bare ca
@@ -11,37 +11,51 @@ mkapicert () {
     | cfssljson -bare apiserver
 }
 
+mketcdcert () {
+    cfssl gencert -ca ca.pem -ca-key ca-key.pem etcd.json \
+    | cfssljson -bare etcd
+}
+
 mkclientcert () {
-    host=$1
+    cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \
-    ip=$2
+    | cfssljson -bare client
-
-    sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
-        | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
-        | cfssljson -bare $host
 }
 
-mkclientcerts () {
+# mkclientcert () {
-    for i in $hosts; do
+#     host=$1
-        IFS=","
+#     ip=$2
-        set -- $i
+
-        mkclientcert $1 10.253.18.$2
+#     sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
-    done
+#         | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
-}
+#         | cfssljson -bare $host
+# }
+
+# mkclientcerts () {
+#     for i in $hosts; do
+#         IFS=","
+#         set -- $i
+#         mkclientcert $1 10.253.18.$2
+#     done
+# }
 
 case $1 in
     all)
         mkcacert
         mkapicert
-        mkclientcerts
+        mketcdcert
+        mkclientcert
         ;;
-    clients)
+    client)
-        mkclientcerts
+        mkclientcert
         ;;
     api)
         mkapicert
         ;;
+    etcd)
+        mketcdcert
+        ;;
     *)
-        echo "usege: mkcerts.sh (all|clients|api)"
+        echo "usege: mkcerts.sh (all|client|api|etcd)"
         exit 1
         ;;
 esac

share/configuration.nix

@@ -1,112 +0,0 @@
-{ config, pkgs, ... }:
-let
-  nix-home = pkgs.callPackage ./nix-home.nix {};
-in
-{
-  # Use the GRUB 2 boot loader.
-  boot.loader.grub.enable = true;
-  boot.loader.grub.version = 2;
-  boot.kernel.sysctl."vm.overcommit_memory"= 1;
-
-  services.vmwareGuest.enable=true;
-
-  # Select internationalisation properties.
-  i18n = {
-     consoleFont = "Lat2-Terminus16";
-     consoleKeyMap = "us";
-     defaultLocale = "en_US.UTF-8";
-  };
-
-  # Set your time zone.
-  time.timeZone = "Europe/Oslo";
-
-  networking.search = [ "itpartner.intern" ];
-  # Enable the OpenSSH daemon.
-  services.openssh.enable = true;
-
-  # The NixOS release to be compatible with for stateful data such as databases.
-  system.stateVersion = "17.09";
-
-  programs.zsh.enable = true;
-  programs.tmux.enable = true;
-
-  virtualisation.docker.enable = true;
-
-  security.sudo.wheelNeedsPassword = false;
-  security.sudo.extraConfig =
-    ''
-      Defaults:root,%wheel env_keep+=LOCALE_ARCHIVE
-      Defaults:root,%wheel env_keep+=NIX_PATH
-      Defaults:root,%wheel env_keep+=TERMINFO_DIRS
-      Defaults env_keep+=SSH_AUTH_SOCK
-      Defaults lecture=never
-      Defaults shell_noargs
-      root   ALL=(ALL) SETENV: ALL
-      %wheel ALL=(ALL) NOPASSWD: ALL, SETENV: ALL
-     '';
-
-  security.rtkit.enable = true;
-
-  fileSystems."/data" = {
-     device = "10.253.18.103:/data";
-     fsType = "nfs";
-   };
-
-  nixpkgs.config.allowUnfree = true;
-  environment.systemPackages =
-    with pkgs;
-    let
-      sys = [
-        stdenv
-        findutils
-        coreutils
-        psmisc
-        iputils
-        nettools
-        netcat
-        rsync
-        htop
-        iotop
-        zsh
-        wget
-        vimNox
-        python
-        file
-        bc
-        sshuttle
-        nix-prefetch-git
-        docker
-        nix-home
-      ];
-      devel = [
-        git
-        patchelf
-        binutils
-        gcc
-      ];
-      dotnet = with dotnetPackages; [
-        fsharp
-        mono-addins
-        mono
-        Fake
-        Paket
-      ];
-      node = with nodePackages; [
-        nodejs
-        npm
-        npm2nix
-        yo
-        gulp
-      ];
-      k8s = [
-        ebtables
-        ethtool
-        socat
-      ];
-    in
-    devel ++
-    dotnet ++
-    node ++
-    k8s ++
-    sys;
-}

share/k8s-node.nix

@@ -1,22 +0,0 @@
-{ config, pkgs, ... }:
-{
-  services.kubernetes.roles = [ "node" ];
-  services.kubernetes.kubeconfig = {
-    server = "https://10.253.18.101:443";
-    # caFile = ../pki/ca.cert.pem;
-    # certFile = ../pki/certificate.pem;
-    # keyFile = ../pki/key.pem;
-  };
-  services.kubernetes.apiserver = {
-    # kubeletClientCaFile = ../pki/ca.cert.pem;
-    # kubeletClientCertFile = ../pki/certificate.pem;
-    # kubeletClientKeyFile = ../pki/key.pem;
-    # clientCaFile = ../pki/ca.cert.pem;
-    tlsCertFile = ../pki/certificate.pem;
-    tlsKeyFile = ../pki/key.pem;
-  };
-  services.kubernetes.kubelet = {
-    tlsCertFile = ../pki/certificate.pem;
-    tlsKeyFile = ../pki/key.pem;
-  };
-}

share/k8s.nix

@@ -1,9 +0,0 @@
-{ config, pkgs, ... }:
-{
-  services.kubernetes.roles = [ "node" ];
-  services.kubernetes.kubeconfig = {
-    server = "https://10.253.18.101:443";
-    caFile = ../pki/ca.pem;
-  };
-  networking.firewall.allowedTCPPorts = [ 10250 ];
-}