oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Bug fixes. Disable FW for now.

Browse Source
This commit is contained in:
Jonas Juselius
2017-07-14 15:25:54 +02:00
parent cd26c7d68c
commit e2c2c6b811
1 changed files with 52 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
k8s.nix
View File

@@ -4,7 +4,7 @@ let
name = "kubernetes";
csr = csr {
cn = "kubernetes";
hosts = ''"kubernetes", "k8s0-0", "etcd0", "10.253.18.100"'';
hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
};
profile = "server";
};
@@ -13,7 +13,7 @@ let
name = "etcd0";
csr = csr {
cn = "etcd0";
hosts = ''"etcd0", "k8s0-0", "10.253.18.100"'';
hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
};
profile = "peer";
};
@@ -22,7 +22,7 @@ let
name = "etcd1";
csr = csr {
cn = "etcd1";
hosts = ''"etcd1", "k8s0-1", "10.253.18.101"'';
hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
};
profile = "peer";
};
@@ -48,10 +48,8 @@ let
client_key = "${client-cert}/cert-key.pem";
client_cert = "${client-cert}/cert.pem";
etcdCluster = [
"etcd0=https://etcd0:2380"
"etcd1=https://etcd1:2380"
];
etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ];
etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ];
etcdConfig = etcd: {
services.etcd = {
@@ -71,9 +69,10 @@ let
ETCDCTL_KEY_FILE = "${etcd.key}";
ETCDCTL_CERT_FILE = "${etcd.cert}";
ETCDCTL_CA_FILE = "${ca_cert}";
ETCDCTL_PEERS = "https://127.0.0.1:2379";
ETCDCTL_PEERS = "https://localhost:2379";
};
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
# networking.firewall.allowedTCPPorts = [ 2379 2380 ];
systemd.services.flannel.after = [ "etcd.service" ];
};
flannelConfig = {
@@ -82,18 +81,16 @@ let
network = "10.10.0.0/16";
iface = "enp2s0";
etcd = {
endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
endpoints = etcdServers;
caFile = "${ca_cert}";
keyFile = "${client_key}";
certFile = "${client_cert}";
};
};
# networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
};
kubeConfig = {
require = [ flannelConfig ];
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
systemd.services.docker = {
after = [ "flannel.service" ];
serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
@@ -101,7 +98,7 @@ let
virtualisation.docker.extraOptions =
"--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
services.kubernetes.etcd = {
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
servers = etcdServers;
caFile = "${ca_cert}";
keyFile = "${client_key}";
certFile = "${client_cert}";
@@ -113,7 +110,7 @@ let
services.kubernetes = {
roles = [ "node" ];
kubeconfig = {
server = "https://kubernetes:443";
server = "https://10.253.18.100:443";
caFile = "${ca_cert}";
keyFile = "${client_key}";
certFile = "${client_cert}";
@@ -122,18 +119,21 @@ let
tlsKeyFile = "${client_key}";
tlsCertFile = "${client_cert}";
networkPlugin = null;
clusterDns = "10.253.18.100";
clusterDns = "10.10.21.0";
};
};
networking.firewall.allowedTCPPorts = [ 53 ];
networking.firewall.allowedUDPPorts = [ 53 ]; # VXLAN
networking.firewall = {
enable = false;
# allowedTCPPorts = [ 53 10250 8000 8080 ];
# allowedUDPPorts = [ 53 ];
# trustedInterfaces = [ "flannel.1" "docker0" ];
# extraCommands = ''
# iptables -P FORWARD ACCEPT
# '';
};
};
kubeMaster = {
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
services.kubernetes = {
roles = [ "master" ];
apiserver = {
@@ -142,26 +142,39 @@ let
clientCaFile = "${ca_cert}";
tlsKeyFile = "${server_key}";
tlsCertFile = "${server_cert}";
# serviceAccountKeyFile = "${server_key}";
# kubeletClientCaFile = "${ca_cert}";
# kubeletClientKeyFile = "${server_key}";
# kubeletClientCertFile = "${server_cert}";
# kubeletClientKeyFile = "${client_key}";
# kubeletClientCertFile = "${client_cert}";
};
scheduler.leaderElect = true;
controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = "${server_key}";
};
networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
systemd.services.flannel.after = [ "etcd.service" ];
# networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
# networking.firewall.allowedUDPPorts = [ 53 ];
};
baseConfig = node: {
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking.hostName = node;
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.100 etcd0 k8s0-0 kubernetes
10.253.18.101 etcd1
'';
virtualisation.docker.enable = true;
};
etcdConf0 = etcdConfig {
name = "etcd0";
key = etcd0_key;
cert = etcd0_cert;
};
etcdConf1 = etcdConfig {
name = "etcd1";
key = etcd1_key;
cert = etcd1_cert;
};
minion = host: ip: { config, lib, pkgs, ... }:
@@ -171,38 +184,32 @@ let
in
{
deployment.targetHost = ip;
require = [ base kubeConfig kubeNode ];
require = [ base flannelConfig kubeConfig kubeNode ];
};
in
{
k8s0-0 = { config, lib, pkgs, ... }:
let
host = "k8s0-0";
base = baseConfig host;
etcd = etcdConfig {
name = "etcd0";
key = etcd0_key;
cert = etcd0_cert;
};
base = baseConfig "k8s0-0";
etcd = etcdConf0;
in
{
deployment.targetHost = "10.253.18.100";
require = [ base etcd kubeConfig kubeMaster kubeNode ];
require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ];
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
};
k8s0-1 = { config, lib, pkgs, ... }:
let
host = "k8s0-1";
base = baseConfig host;
etcd = etcdConfig {
name = "etcd1";
key = etcd1_key;
cert = etcd1_cert;
};
base = baseConfig "k8s0-1";
etcd = etcdConf1;
in
{
deployment.targetHost = "10.253.18.101";
require = [ base etcd kubeConfig kubeNode ];
require = [ base etcd flannelConfig kubeConfig kubeNode ];
};
k8s0-2 = minion "k8s0-2" "10.253.18.102";