oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Works!

Browse Source
This commit is contained in:
Jonas Juselius
2017-08-18 11:31:07 +02:00
parent e2c2c6b811
commit 85a045577d
13 changed files with 458 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
k8s.nix
View File

@@ -1,107 +1,57 @@
with import ./base/pki.nix;
with import ./certs.nix;
let
server-cert = mkCert {
name = "kubernetes";
csr = csr {
cn = "kubernetes";
hosts = ''"kubernetes", "k8s0-0", "etcd0", "localhost", "10.253.18.100"'';
};
profile = "server";
};
etcdServers = [ "etcd0" "etcd1" "etcd2" ];
# etcdServers = [ "k8s0-0" "k8s0-1" "k8s0-2" ];
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers;
etcd0-cert = mkCert {
name = "etcd0";
csr = csr {
cn = "etcd0";
hosts = ''"etcd0", "k8s0-0", "localhost", "10.253.18.100"'';
};
profile = "peer";
};
etcd1-cert = mkCert {
name = "etcd1";
csr = csr {
cn = "etcd1";
hosts = ''"etcd1", "k8s0-1", "localhost", "10.253.18.101"'';
};
profile = "peer";
};
client-cert = mkCert {
name = "client";
csr = csr {
cn = "client";
hosts = '''';
};
profile = "client";
};
server_key = "${server-cert}/cert-key.pem";
server_cert = "${server-cert}/cert.pem";
etcd0_key = "${etcd0-cert}/cert-key.pem";
etcd0_cert = "${etcd0-cert}/cert.pem";
etcd1_key = "${etcd1-cert}/cert-key.pem";
etcd1_cert = "${etcd1-cert}/cert.pem";
client_key = "${client-cert}/cert-key.pem";
client_cert = "${client-cert}/cert.pem";
etcdServers = [ "https://etcd0:2379" "https://etcd1:2379" ];
etcdCluster = [ "etcd0=https://etcd0:2380" "etcd1=https://etcd1:2380" ];
etcdConfig = etcd: {
etcdConfig = name: {
services.etcd = {
name = etcd.name;
inherit name;
enable = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
peerClientCertAuth = true;
keyFile = "${etcd.key}";
certFile = "${etcd.cert}";
trustedCaFile = "${ca_cert}";
advertiseClientUrls = [ "https://${etcd.name}:2379" ];
initialAdvertisePeerUrls = [ "https://${etcd.name}:2380" ];
keyFile = etcd_key;
certFile = etcd_cert;
trustedCaFile = ca_pem;
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
initialCluster = etcdCluster;
};
environment.variables = {
ETCDCTL_KEY_FILE = "${etcd.key}";
ETCDCTL_CERT_FILE = "${etcd.cert}";
ETCDCTL_CA_FILE = "${ca_cert}";
ETCDCTL_PEERS = "https://localhost:2379";
ETCDCTL_KEY_FILE = "${etcd_client_key}";
ETCDCTL_CERT_FILE = "${etcd_client_cert}";
ETCDCTL_CA_FILE = "${ca_pem}";
ETCDCTL_PEERS = "https://127.0.0.1:2379";
};
# networking.firewall.allowedTCPPorts = [ 2379 2380 ];
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
systemd.services.flannel.after = [ "etcd.service" ];
};
flannelConfig = {
kubeConfig = {
services.flannel = {
enable = true;
network = "10.10.0.0/16";
iface = "enp2s0";
iface = "ens32";
etcd = {
endpoints = etcdServers;
caFile = "${ca_cert}";
keyFile = "${client_key}";
certFile = "${client_cert}";
endpoints = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
};
# networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
};
kubeConfig = {
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
systemd.services.docker = {
after = [ "flannel.service" ];
serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
};
virtualisation.docker.extraOptions =
"--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU";
services.kubernetes.etcd = {
servers = etcdServers;
caFile = "${ca_cert}";
keyFile = "${client_key}";
certFile = "${client_cert}";
servers = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
# services.kubernetes.verbose = true;
};
@@ -110,26 +60,26 @@ let
services.kubernetes = {
roles = [ "node" ];
kubeconfig = {
server = "https://10.253.18.100:443";
caFile = "${ca_cert}";
keyFile = "${client_key}";
certFile = "${client_cert}";
server = "https://10.253.18.100:4443";
keyFile = worker_key;
certFile = worker_cert;
caFile = ca_pem;
};
kubelet = {
tlsKeyFile = "${client_key}";
tlsCertFile = "${client_cert}";
tlsKeyFile = worker_key;
tlsCertFile = worker_cert;
networkPlugin = null;
clusterDns = "10.10.21.0";
clusterDns = "10.253.18.100";
};
};
networking.firewall = {
enable = false;
# allowedTCPPorts = [ 53 10250 8000 8080 ];
# allowedUDPPorts = [ 53 ];
# trustedInterfaces = [ "flannel.1" "docker0" ];
# extraCommands = ''
# iptables -P FORWARD ACCEPT
# '';
networking = {
firewall = {
enable = true;
# trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
allowedTCPPorts = [ 53 10250 ];
allowedUDPPorts = [ 53 ];
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
};
};
};
@@ -137,44 +87,46 @@ let
services.kubernetes = {
roles = [ "master" ];
apiserver = {
publicAddress = "0.0.0.0";
address = "0.0.0.0";
clientCaFile = "${ca_cert}";
tlsKeyFile = "${server_key}";
tlsCertFile = "${server_cert}";
# serviceAccountKeyFile = "${server_key}";
# kubeletClientCaFile = "${ca_cert}";
# kubeletClientKeyFile = "${client_key}";
# kubeletClientCertFile = "${client_cert}";
publicAddress = "0.0.0.0";
advertiseAddress = "10.253.18.100";
securePort = 4443;
tlsKeyFile = apiserver_key;
tlsCertFile = apiserver_cert;
clientCaFile = ca_pem;
kubeletClientCaFile = ca_pem;
kubeletClientKeyFile = worker_key;
kubeletClientCertFile = worker_cert;
# serviceAccountKeyFile = apiserver_key;
};
scheduler.leaderElect = true;
controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = "${server_key}";
controllerManager.serviceAccountKeyFile = apiserver_key;
dns.enable = true;
dns.port = 4053;
};
networking.firewall = {
allowedTCPPorts = [ 5000 8080 4443 4053 ];
allowedUDPPorts = [ 4053 ];
};
# networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
# networking.firewall.allowedUDPPorts = [ 53 ];
};
baseConfig = node: {
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking.hostName = node;
networking.extraHosts = ''
10.253.18.100 etcd0 k8s0-0 kubernetes
10.253.18.101 etcd1
'';
virtualisation.docker.enable = true;
};
etcdConf0 = etcdConfig {
name = "etcd0";
key = etcd0_key;
cert = etcd0_cert;
};
etcdConf1 = etcdConfig {
name = "etcd1";
key = etcd1_key;
cert = etcd1_cert;
networking = {
hostName = node;
extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
10.253.18.102 etcd2
'';
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
firewall.allowedTCPPorts = [ 80 443 ];
};
services.dnsmasq.enable = true;
services.dnsmasq.servers = [
"/cluster.local/10.253.18.100#4053"
];
};
minion = host: ip: { config, lib, pkgs, ... }:
@@ -184,18 +136,19 @@ let
in
{
deployment.targetHost = ip;
require = [ base flannelConfig kubeConfig kubeNode ];
};
require = [ base kubeConfig kubeNode ];
services.kubernetes.dns.enable = false;
};
in
{
k8s0-0 = { config, lib, pkgs, ... }:
let
base = baseConfig "k8s0-0";
etcd = etcdConf0;
etcd = etcdConfig "etcd0";
in
{
deployment.targetHost = "10.253.18.100";
require = [ base etcd flannelConfig kubeConfig kubeMaster kubeNode ];
require = [ base etcd kubeConfig kubeMaster kubeNode ];
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
@@ -205,12 +158,22 @@ in
k8s0-1 = { config, lib, pkgs, ... }:
let
base = baseConfig "k8s0-1";
etcd = etcdConf1;
etcd = etcdConfig "etcd1";
in
{
deployment.targetHost = "10.253.18.101";
require = [ base etcd flannelConfig kubeConfig kubeNode ];
};
require = [ base etcd kubeConfig kubeNode ];
services.kubernetes.dns.enable = false;
};
k8s0-2 = minion "k8s0-2" "10.253.18.102";
k8s0-2 = { config, lib, pkgs, ... }:
let
base = baseConfig "k8s0-2";
etcd = etcdConfig "etcd2";
in
{
deployment.targetHost = "10.253.18.102";
require = [ base etcd kubeConfig kubeNode ];
services.kubernetes.dns.enable = false;
};
}