oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Bug fixes. Disable FW for now.

Browse Source
This commit is contained in:
Jonas Juselius
2017-07-14 15:25:54 +02:00
parent cd26c7d68c
commit e2c2c6b811
1 changed files with 52 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
k8s.nix
View File

@@ -4,7 +4,7 @@ let
name = "kubernetes"; name = "kubernetes";
csr = csr { csr = csr {
cn = "kubernetes"; cn = "kubernetes";
hosts = ''"kubernetes", "k8s0-0", "etcd0", "10.253.18.100"''; hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
}; };
profile = "server"; profile = "server";
}; };
@@ -13,7 +13,7 @@ let
name = "etcd0"; name = "etcd0";
csr = csr { csr = csr {
cn = "etcd0"; cn = "etcd0";
hosts = ''"etcd0", "k8s0-0", "10.253.18.100"''; hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
}; };
profile = "peer"; profile = "peer";
}; };
@@ -22,7 +22,7 @@ let
name = "etcd1"; name = "etcd1";
csr = csr { csr = csr {
cn = "etcd1"; cn = "etcd1";
hosts = ''"etcd1", "k8s0-1", "10.253.18.101"''; hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
}; };
profile = "peer"; profile = "peer";
}; };
@@ -48,10 +48,8 @@ let
client_key = "${client-cert}/cert-key.pem"; client_key = "${client-cert}/cert-key.pem";
client_cert = "${client-cert}/cert.pem"; client_cert = "${client-cert}/cert.pem";
etcdCluster = [ etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ];
"etcd0=https://etcd0:2380" etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ];
"etcd1=https://etcd1:2380"
];
etcdConfig = etcd: { etcdConfig = etcd: {
services.etcd = { services.etcd = {
@@ -71,9 +69,10 @@ let
ETCDCTL_KEY_FILE = "${etcd.key}"; ETCDCTL_KEY_FILE = "${etcd.key}";
ETCDCTL_CERT_FILE = "${etcd.cert}"; ETCDCTL_CERT_FILE = "${etcd.cert}";
ETCDCTL_CA_FILE = "${ca_cert}"; ETCDCTL_CA_FILE = "${ca_cert}";
ETCDCTL_PEERS = "https://127.0.0.1:2379"; ETCDCTL_PEERS = "https://localhost:2379";
}; };
networking.firewall.allowedTCPPorts = [ 2379 2380 ]; # networking.firewall.allowedTCPPorts = [ 2379 2380 ];
systemd.services.flannel.after = [ "etcd.service" ];
}; };
flannelConfig = { flannelConfig = {
@@ -82,18 +81,16 @@ let
network = "10.10.0.0/16"; network = "10.10.0.0/16";
iface = "enp2s0"; iface = "enp2s0";
etcd = { etcd = {
endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ]; endpoints = etcdServers;
caFile = "${ca_cert}"; caFile = "${ca_cert}";
keyFile = "${client_key}"; keyFile = "${client_key}";
certFile = "${client_cert}"; certFile = "${client_cert}";
}; };
}; };
# networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
}; };
kubeConfig = { kubeConfig = {
require = [ flannelConfig ];
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
systemd.services.docker = { systemd.services.docker = {
after = [ "flannel.service" ]; after = [ "flannel.service" ];
serviceConfig.EnvironmentFile = "/run/flannel/subnet.env"; serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
@@ -101,7 +98,7 @@ let
virtualisation.docker.extraOptions = virtualisation.docker.extraOptions =
"--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET"; "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
services.kubernetes.etcd = { services.kubernetes.etcd = {
servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; servers = etcdServers;
caFile = "${ca_cert}"; caFile = "${ca_cert}";
keyFile = "${client_key}"; keyFile = "${client_key}";
certFile = "${client_cert}"; certFile = "${client_cert}";
@@ -113,7 +110,7 @@ let
services.kubernetes = { services.kubernetes = {
roles = [ "node" ]; roles = [ "node" ];
kubeconfig = { kubeconfig = {
server = "https://kubernetes:443"; server = "https://10.253.18.100:443";
caFile = "${ca_cert}"; caFile = "${ca_cert}";
keyFile = "${client_key}"; keyFile = "${client_key}";
certFile = "${client_cert}"; certFile = "${client_cert}";
@@ -122,18 +119,21 @@ let
tlsKeyFile = "${client_key}"; tlsKeyFile = "${client_key}";
tlsCertFile = "${client_cert}"; tlsCertFile = "${client_cert}";
networkPlugin = null; networkPlugin = null;
clusterDns = "10.253.18.100"; clusterDns = "10.10.21.0";
}; };
}; };
networking.firewall.allowedTCPPorts = [ 53 ]; networking.firewall = {
networking.firewall.allowedUDPPorts = [ 53 ]; # VXLAN enable = false;
# allowedTCPPorts = [ 53 10250 8000 8080 ];
# allowedUDPPorts = [ 53 ];
# trustedInterfaces = [ "flannel.1" "docker0" ];
# extraCommands = ''
# iptables -P FORWARD ACCEPT
# '';
};
}; };
kubeMaster = { kubeMaster = {
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
services.kubernetes = { services.kubernetes = {
roles = [ "master" ]; roles = [ "master" ];
apiserver = { apiserver = {
@@ -142,26 +142,39 @@ let
clientCaFile = "${ca_cert}"; clientCaFile = "${ca_cert}";
tlsKeyFile = "${server_key}"; tlsKeyFile = "${server_key}";
tlsCertFile = "${server_cert}"; tlsCertFile = "${server_cert}";
# serviceAccountKeyFile = "${server_key}";
# kubeletClientCaFile = "${ca_cert}"; # kubeletClientCaFile = "${ca_cert}";
# kubeletClientKeyFile = "${server_key}"; # kubeletClientKeyFile = "${client_key}";
# kubeletClientCertFile = "${server_cert}"; # kubeletClientCertFile = "${client_cert}";
}; };
scheduler.leaderElect = true; scheduler.leaderElect = true;
controllerManager.leaderElect = true; controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = "${server_key}"; controllerManager.serviceAccountKeyFile = "${server_key}";
}; };
# networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ]; # networking.firewall.allowedUDPPorts = [ 53 ];
systemd.services.flannel.after = [ "etcd.service" ];
}; };
baseConfig = node: { baseConfig = node: {
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking.hostName = node; networking.hostName = node;
networking.extraHosts = '' networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes 10.253.18.100 etcd0 k8s0-0 kubernetes
10.253.18.101 etcd1 10.253.18.101 etcd1
''; '';
virtualisation.docker.enable = true;
};
etcdConf0 = etcdConfig {
name = "etcd0";
key = etcd0_key;
cert = etcd0_cert;
};
etcdConf1 = etcdConfig {
name = "etcd1";
key = etcd1_key;
cert = etcd1_cert;
}; };
minion = host: ip: { config, lib, pkgs, ... }: minion = host: ip: { config, lib, pkgs, ... }:
@@ -171,38 +184,32 @@ let
in in
{ {
deployment.targetHost = ip; deployment.targetHost = ip;
require = [ base kubeConfig kubeNode ]; require = [ base flannelConfig kubeConfig kubeNode ];
}; };
in in
{ {
k8s0-0 = { config, lib, pkgs, ... }: k8s0-0 = { config, lib, pkgs, ... }:
let let
host = "k8s0-0"; base = baseConfig "k8s0-0";
base = baseConfig host; etcd = etcdConf0;
etcd = etcdConfig {
name = "etcd0";
key = etcd0_key;
cert = etcd0_cert;
};
in in
{ {
deployment.targetHost = "10.253.18.100"; deployment.targetHost = "10.253.18.100";
require = [ base etcd kubeConfig kubeMaster kubeNode ]; require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ];
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
}; };
k8s0-1 = { config, lib, pkgs, ... }: k8s0-1 = { config, lib, pkgs, ... }:
let let
host = "k8s0-1"; base = baseConfig "k8s0-1";
base = baseConfig host; etcd = etcdConf1;
etcd = etcdConfig {
name = "etcd1";
key = etcd1_key;
cert = etcd1_cert;
};
in in
{ {
deployment.targetHost = "10.253.18.101"; deployment.targetHost = "10.253.18.101";
require = [ base etcd kubeConfig kubeNode ]; require = [ base etcd flannelConfig kubeConfig kubeNode ];
}; };
k8s0-2 = minion "k8s0-2" "10.253.18.102"; k8s0-2 = minion "k8s0-2" "10.253.18.102";