Major revamp and restructuring of k8s setup:

* replicated etcd setup for redundancy
* use Flannel network overlay
* naming conventions
* new repo structure
* etc.

k8s.nix

@@ -1,19 +1,129 @@
-{
-  k8s00 = { config, lib, pkgs, ... }:
-    {
-      deployment.targetHost = "10.253.18.101";
-      imports = [ ./k8s00/configuration.nix ];
+let
+  etcdConfig = name: {
+    services.etcd = {
+      inherit name;
+      advertiseClientUrls = [ "https://${name}:2379" ];
+      initialAdvertisePeerUrls = [ "https://${name}:2380" ];
+      enable = true;
+      certFile = ./pki/etcd.pem;
+      keyFile = ./pki/etcd-key.pem;
+      trustedCaFile = ./pki/ca.pem;
+      peerClientCertAuth = true;
+      listenClientUrls = ["https://0.0.0.0:2379"];
+      listenPeerUrls = ["https://0.0.0.0:2380"];
+      initialCluster = [
+        "etcd0=https://etcd0:2380"
+        "etcd1=https://etcd1:2380"
+      ];
+      # environment.variables = {
+        #   ETCDCTL_CERT_FILE = ./pki/client.pem;
+        #   ETCDCTL_KEY_FILE = ./pki/client-key.pem;
+        #   ETCDCTL_CA_FILE = ./pki/ca.pem;
+        #   ETCDCTL_PEERS = "https://127.0.0.1:2379";
+        # };
+    };
+    networking.firewall.allowedTCPPorts = [ 2379 2380 ];
+  };
+
+  flannelConfig = {
+    services.flannel = {
+      enable = true;
+      network = "10.10.0.0/16";
+      iface = "enp0s3";
+      etcd = {
+        endpoints = ["https://etcd0:2379" "https://etcd1:2379" ];
+        certFile = ./pki/client.pem;
+        keyFile = ./pki/client-key.pem;
+        caFile = ./pki/ca.pem;
+      };
+    };
+  };
+
+  kubeNode = {
+    services.kubernetes = {
+      # verbose = true;
+      roles = [ "node" ];
+      kubeconfig = {
+        server = "https://kubernetes:443";
+        caFile = ./pki/ca.pem;
+        certFile = ./pki/client.pem;
+        keyFile = ./pki/client-key.pem;
+      };
+      etcd = {
+        servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
+        certFile = ./pki/client.pem;
+        keyFile = ./pki/client-key.pem;
+        caFile = ./pki/ca.pem;
+      };
+      # kubelet.clusterDns = "10.10.1.1";
     };
 
-  k8s01 = { config, lib, pkgs, ... }:
+    networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
+    networking.firewall.allowedTCPPorts = [ 10250 ];
+    networking.extraHosts = ''
+          10.253.18.100 etcd0 kubernetes
+          10.253.18.101 etcd1
+    '';
+
+    systemd.services.docker.after = [ "flannel.service" ];
+    systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
+    virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
+  };
+
+  kubeMaster = {
+    services.dockerRegistry = {
+      enable = true;
+      listenAddress = "0.0.0.0";
+    };
+
+    services.kubernetes = {
+      roles = [ "master" ];
+      apiserver = {
+        publicAddress = "0.0.0.0";
+        address = "0.0.0.0";
+        clientCaFile = ./pki/ca.pem;
+        tlsCertFile = ./pki/apiserver.pem;
+        tlsKeyFile = ./pki/apiserver-key.pem;
+        kubeletClientCaFile = ./pki/ca.pem;
+        kubeletClientCertFile = ./pki/client.pem;
+        kubeletClientKeyFile = ./pki/client-key.pem;
+      };
+      scheduler.leaderElect = true;
+      controllerManager.leaderElect = true;
+      controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
+    };
+
+    networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
+    networking.firewall.allowedUDPPorts = [ 53 ];
+    systemd.services.flannel.after = [ "etcd.service" ];
+  };
+in
+{
+  k8s0-0 = { config, lib, pkgs, ... }:
+    let etcd = etcdConfig "etcd0"; in
     {
       deployment.targetHost = "10.253.18.100";
-      imports = [ ./k8s01/configuration.nix ];
+      networking.hostName = "k8s0-0";
+      imports = [ ./hw/k8s0-0.nix ./base/configuration.nix ];
+      require = [ etcd flannelConfig ];
+      # require = [ etcd flannelConfig kubeMaster kubeNode ];
     };
 
-  k8s02 = { config, lib, pkgs, ... }:
+  k8s0-1 = { config, lib, pkgs, ... }:
+    let etcd = etcdConfig "etcd1"; in
+    {
+      deployment.targetHost = "10.253.18.101";
+      networking.hostName = "k8s0-1";
+      imports = [ ./hw/k8s0-1.nix ./base/configuration.nix ];
+      require = [ etcd flannelConfig  ];
+      # require = [ etcd flannelConfig kubeNode ];
+  };
+
+  k8s0-2 = { config, lib, pkgs, ... }:
     {
       deployment.targetHost = "10.253.18.102";
-      imports = [ ./k8s02/configuration.nix ];
-    };
+      networking.hostName = "k8s0-2";
+      imports = [ ./hw/k8s0-2.nix ./base/configuration.nix ];
+      require = [ flannelConfig  ];
+  };
 }