oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Lots of small fixes. Now it works!

Browse Source
This commit is contained in:
Jonas Juselius
2017-07-10 13:40:09 +02:00
parent bacb4ff2dd
commit c21abb5a5f
5 changed files with 98 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
k8s.nix
View File

@@ -2,80 +2,86 @@ let
etcdConfig = name: {
services.etcd = {
inherit name;
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
enable = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
peerClientCertAuth = true;
certFile = ./pki/etcd.pem;
keyFile = ./pki/etcd-key.pem;
trustedCaFile = ./pki/ca.pem;
peerClientCertAuth = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
initialCluster = [
"etcd0=https://etcd0:2380"
"etcd1=https://etcd1:2380"
];
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki/client.pem;
# ETCDCTL_KEY_FILE = ./pki/client-key.pem;
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
};
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki + "/${name}.pem";
# ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem";
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
};
flannelConfig = {
flannelConfig = node: {
services.flannel = {
enable = true;
network = "10.10.0.0/16";
iface = "enp0s3";
iface = "enp2s0";
etcd = {
endpoints = ["https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
caFile = ./pki/ca.pem;
};
};
};
kubeNode = {
etcdClient = node:{
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
caFile = ./pki/ca.pem;
};
kubeConfig = node: {
require = [ (flannelConfig node) ];
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
# services.kubernetes.verbose = true;
};
kubeNode = doConfig: node: {
require = if doConfig then [ (kubeConfig node) ] else [];
services.kubernetes = {
# verbose = true;
roles = [ "node" ];
kubeconfig = {
server = "https://kubernetes:443";
caFile = ./pki/ca.pem;
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
};
etcd = {
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
caFile = ./pki/ca.pem;
kubelet = {
tlsCertFile = ./pki + "/${node}.pem";
tlsKeyFile = ./pki + "/${node}-key.pem";
networkPlugin = null;
clusterDns = "10.253.18.100";
};
# kubelet.clusterDns = "10.10.1.1";
etcd = if doConfig then (etcdClient node) else {};
};
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
};
kubeMaster = {
kubeMaster = node: {
require = [ (kubeConfig node) (kubeNode false node)];
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
services.kubernetes = {
roles = [ "master" ];
apiserver = {
@@ -84,10 +90,11 @@ let
clientCaFile = ./pki/ca.pem;
tlsCertFile = ./pki/apiserver.pem;
tlsKeyFile = ./pki/apiserver-key.pem;
kubeletClientCaFile = ./pki/ca.pem;
kubeletClientCertFile = ./pki/client.pem;
kubeletClientKeyFile = ./pki/client-key.pem;
# kubeletClientCaFile = ./pki/ca.pem;
# kubeletClientCertFile = ./pki + "/${node}.pem";
# kubeletClientKeyFile = ./pki + "/${node}-key.pem";
};
etcd = (etcdClient node);
scheduler.leaderElect = true;
controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
@@ -98,9 +105,13 @@ let
systemd.services.flannel.after = [ "etcd.service" ];
};
baseConfig = name: {
networking.hostName = name;
imports = [ "./hw/${name}.nix" ./base/configuration.nix ];
baseConfig = node: {
networking.hostName = node;
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
};
in
{
@@ -108,31 +119,31 @@ in
let
etcd = etcdConfig "etcd0";
base = baseConfig "k8s0-0";
master = kubeMaster "k8s0-0";
in
{
deployment.targetHost = "10.253.18.100";
require = [ base etcd flannelConfig ];
# require = [ base etcd flannelConfig kubeMaster kubeNode ];
require = [ base etcd master ];
};
k8s0-1 = { config, lib, pkgs, ... }:
let
etcd = etcdConfig "etcd1";
base = baseConfig "k8s0-1";
node = kubeNode true "k8s0-1";
in
{
deployment.targetHost = "10.253.18.101";
require = [ base etcd flannelConfig ];
# require = [ base etcd flannelConfig kubeNode ];
require = [ base etcd node ];
};
k8s0-2 = { config, lib, pkgs, ... }:
let
base = baseConfig "k8s0-2";
node = kubeNode true "k8s0-2";
in
{
deployment.targetHost = "10.253.18.102";
require = [ base flannelConfig ];
# require = [ base flannelConfig kubeNode ];
require = [ base node ];
};
}