From e2c2c6b81157185e9c2f7c23ad5e8235047303da Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Fri, 14 Jul 2017 15:25:54 +0200 Subject: [PATCH] Bug fixes. Disable FW for now. --- k8s.nix | 97 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/k8s.nix b/k8s.nix index 5366849..6eb9273 100644 --- a/k8s.nix +++ b/k8s.nix @@ -4,7 +4,7 @@ let name = "kubernetes"; csr = csr { cn = "kubernetes"; - hosts = ''"kubernetes", "k8s0-0", "etcd0", "10.253.18.100"''; + hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"''; }; profile = "server"; }; @@ -13,7 +13,7 @@ let name = "etcd0"; csr = csr { cn = "etcd0"; - hosts = ''"etcd0", "k8s0-0", "10.253.18.100"''; + hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"''; }; profile = "peer"; }; @@ -22,7 +22,7 @@ let name = "etcd1"; csr = csr { cn = "etcd1"; - hosts = ''"etcd1", "k8s0-1", "10.253.18.101"''; + hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"''; }; profile = "peer"; }; @@ -48,10 +48,8 @@ let client_key = "${client-cert}/cert-key.pem"; client_cert = "${client-cert}/cert.pem"; - etcdCluster = [ - "etcd0=https://etcd0:2380" - "etcd1=https://etcd1:2380" - ]; + etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ]; + etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ]; etcdConfig = etcd: { services.etcd = { @@ -71,9 +69,10 @@ let ETCDCTL_KEY_FILE = "${etcd.key}"; ETCDCTL_CERT_FILE = "${etcd.cert}"; ETCDCTL_CA_FILE = "${ca_cert}"; - ETCDCTL_PEERS = "https://127.0.0.1:2379"; + ETCDCTL_PEERS = "https://localhost:2379"; }; - networking.firewall.allowedTCPPorts = [ 2379 2380 ]; + # networking.firewall.allowedTCPPorts = [ 2379 2380 ]; + systemd.services.flannel.after = [ "etcd.service" ]; }; flannelConfig = { @@ -82,18 +81,16 @@ let network = "10.10.0.0/16"; iface = "enp2s0"; etcd = { - endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ]; + endpoints = etcdServers; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; }; }; + # networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN }; kubeConfig = { - require = [ flannelConfig ]; - networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN - networking.firewall.allowedTCPPorts = [ 10250 ]; systemd.services.docker = { after = [ "flannel.service" ]; serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; @@ -101,7 +98,7 @@ let virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; services.kubernetes.etcd = { - servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; + servers = etcdServers; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; @@ -113,7 +110,7 @@ let services.kubernetes = { roles = [ "node" ]; kubeconfig = { - server = "https://kubernetes:443"; + server = "https://10.253.18.100:443"; caFile = "${ca_cert}"; keyFile = "${client_key}"; certFile = "${client_cert}"; @@ -122,18 +119,21 @@ let tlsKeyFile = "${client_key}"; tlsCertFile = "${client_cert}"; networkPlugin = null; - clusterDns = "10.253.18.100"; + clusterDns = "10.10.21.0"; }; }; - networking.firewall.allowedTCPPorts = [ 53 ]; - networking.firewall.allowedUDPPorts = [ 53 ]; # VXLAN + networking.firewall = { + enable = false; + # allowedTCPPorts = [ 53 10250 8000 8080 ]; + # allowedUDPPorts = [ 53 ]; + # trustedInterfaces = [ "flannel.1" "docker0" ]; + # extraCommands = '' + # iptables -P FORWARD ACCEPT + # ''; + }; }; kubeMaster = { - services.dockerRegistry = { - enable = true; - listenAddress = "0.0.0.0"; - }; services.kubernetes = { roles = [ "master" ]; apiserver = { @@ -142,26 +142,39 @@ let clientCaFile = "${ca_cert}"; tlsKeyFile = "${server_key}"; tlsCertFile = "${server_cert}"; + # serviceAccountKeyFile = "${server_key}"; # kubeletClientCaFile = "${ca_cert}"; - # kubeletClientKeyFile = "${server_key}"; - # kubeletClientCertFile = "${server_cert}"; + # kubeletClientKeyFile = "${client_key}"; + # kubeletClientCertFile = "${client_cert}"; }; scheduler.leaderElect = true; controllerManager.leaderElect = true; controllerManager.serviceAccountKeyFile = "${server_key}"; }; - - networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; - systemd.services.flannel.after = [ "etcd.service" ]; + # networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; + # networking.firewall.allowedUDPPorts = [ 53 ]; }; baseConfig = node: { imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; networking.hostName = node; networking.extraHosts = '' - 10.253.18.100 etcd0 kubernetes + 10.253.18.100 etcd0 k8s0-0 kubernetes 10.253.18.101 etcd1 ''; + virtualisation.docker.enable = true; + }; + + etcdConf0 = etcdConfig { + name = "etcd0"; + key = etcd0_key; + cert = etcd0_cert; + }; + + etcdConf1 = etcdConfig { + name = "etcd1"; + key = etcd1_key; + cert = etcd1_cert; }; minion = host: ip: { config, lib, pkgs, ... }: @@ -171,38 +184,32 @@ let in { deployment.targetHost = ip; - require = [ base kubeConfig kubeNode ]; + require = [ base flannelConfig kubeConfig kubeNode ]; }; in { k8s0-0 = { config, lib, pkgs, ... }: let - host = "k8s0-0"; - base = baseConfig host; - etcd = etcdConfig { - name = "etcd0"; - key = etcd0_key; - cert = etcd0_cert; - }; + base = baseConfig "k8s0-0"; + etcd = etcdConf0; in { deployment.targetHost = "10.253.18.100"; - require = [ base etcd kubeConfig kubeMaster kubeNode ]; + require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ]; + services.dockerRegistry = { + enable = true; + listenAddress = "0.0.0.0"; + }; }; k8s0-1 = { config, lib, pkgs, ... }: let - host = "k8s0-1"; - base = baseConfig host; - etcd = etcdConfig { - name = "etcd1"; - key = etcd1_key; - cert = etcd1_cert; - }; + base = baseConfig "k8s0-1"; + etcd = etcdConf1; in { deployment.targetHost = "10.253.18.101"; - require = [ base etcd kubeConfig kubeNode ]; + require = [ base etcd flannelConfig kubeConfig kubeNode ]; }; k8s0-2 = minion "k8s0-2" "10.253.18.102";