oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add git01 to k8s cluster pool.

Browse Source
This commit is contained in:
Jonas Juselius
2017-10-26 12:01:52 +02:00
parent cb74a6d72f
commit 336ecccd09
2 changed files with 74 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
certs.nix
View File

@@ -51,7 +51,7 @@ let
DNS.3 = kubernetes.default.svc DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local DNS.4 = kubernetes.default.svc.cluster.local
DNS.4 = k8s0-0.itpartner.no DNS.4 = k8s0-0.itpartner.no
IP.1 = 10.10.10.1 IP.1 = 10.0.0.1
IP.2 = 10.253.18.100 IP.2 = 10.253.18.100
''; '';
@@ -65,11 +65,12 @@ let
keyUsage = nonRepudiation, digitalSignature, keyEncipherment keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names subjectAltName = @alt_names
[alt_names] [alt_names]
DNS.1 = k8s0-0 DNS.1 = *.itpartner.no
DNS.2 = k8s0-1 DNS.2 = *.itpartner.intern
DNS.3 = k8s0-2 DNS.3 = k8s0-0
DNS.4 = *.itpartner.no DNS.4 = k8s0-1
DNS.5 = *.itpartner.intern DNS.5 = k8s0-2
DNS.6 = git01
''; '';
ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048"; ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";

110
k8s.nix
View File

@@ -2,6 +2,7 @@ with import ./certs.nix;
let let
pkgs = import <nixpkgs> {}; pkgs = import <nixpkgs> {};
kube_apiserver = "https://10.253.18.100:443";
etcdServers = [ "etcd0" "etcd1" "etcd2" ]; etcdServers = [ "etcd0" "etcd1" "etcd2" ];
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers; etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers; etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers;
@@ -30,55 +31,38 @@ let
systemd.services.flannel.after = [ "etcd.service" ]; systemd.services.flannel.after = [ "etcd.service" ];
}; };
kubeConfig = { kubeconfig = {
systemd.services.kubelet.path = [ pkgs.socat ]; caFile = ca_pem;
services.flannel = { keyFile = worker_key;
enable = true; certFile = worker_cert;
network = "10.10.0.0/16"; server = kube_apiserver;
iface = "ens32";
etcd = {
endpoints = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
};
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
systemd.services.docker = {
after = [ "flannel.service" ];
serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
};
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU";
services.kubernetes.etcd = {
servers = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
# services.kubernetes.verbose = true;
}; };
kubeNode = { kubeNode = {
services.kubernetes = { services.kubernetes = {
roles = [ "node" ]; roles = [ "node" ];
kubeconfig = { kubeconfig = {
server = "https://10.253.18.100:4443"; server = kube_apiserver;
keyFile = worker_key; keyFile = worker_key;
certFile = worker_cert; certFile = worker_cert;
caFile = ca_pem; caFile = ca_pem;
}; };
kubelet = { kubelet = {
tlsKeyFile = worker_key; enable = true;
tlsCertFile = worker_cert; clientCaFile = ca_pem;
tlsKeyFile = worker_key;
tlsCertFile = worker_cert;
networkPlugin = null; networkPlugin = null;
clusterDns = "10.253.18.100"; # clusterDns = "10.253.18.100";
clusterDns = "10.0.0.254";
inherit kubeconfig;
}; };
}; };
networking = { networking = {
firewall = { firewall = {
enable = true; enable = true;
# trustedInterfaces = [ "flannel.1" "docker0" "veth+" ]; # trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
allowedTCPPorts = [ 53 10250 ]; allowedTCPPorts = [ 53 4194 10250 ];
allowedUDPPorts = [ 53 ]; allowedUDPPorts = [ 53 ];
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE''; extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
}; };
@@ -89,11 +73,12 @@ let
kubeMaster = { kubeMaster = {
services.kubernetes = { services.kubernetes = {
roles = [ "master" ]; roles = [ "master" ];
kubelet.unschedulable = true;
apiserver = { apiserver = {
address = "0.0.0.0"; address = "0.0.0.0";
publicAddress = "0.0.0.0"; publicAddress = "0.0.0.0";
advertiseAddress = "10.253.18.100"; advertiseAddress = "10.253.18.100";
securePort = 4443; securePort = 443;
tlsKeyFile = apiserver_key; tlsKeyFile = apiserver_key;
tlsCertFile = apiserver_cert; tlsCertFile = apiserver_cert;
clientCaFile = ca_pem; clientCaFile = ca_pem;
@@ -103,19 +88,40 @@ let
serviceAccountKeyFile = apiserver_key; serviceAccountKeyFile = apiserver_key;
}; };
scheduler.leaderElect = true; scheduler.leaderElect = true;
controllerManager.leaderElect = true; controllerManager = {
controllerManager.serviceAccountKeyFile = apiserver_key; leaderElect = true;
controllerManager.rootCaFile = ca_pem; serviceAccountKeyFile = apiserver_key;
dns.enable = true; rootCaFile = ca_pem;
dns.port = 4053; inherit kubeconfig;
};
addons.dashboard.enable = true;
addons.dns.enable = true;
}; };
networking.firewall = { networking.firewall = {
allowedTCPPorts = [ 5000 8080 4443 4053 ]; allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
allowedUDPPorts = [ 4053 ]; # allowedUDPPorts = [ 4053 ];
}; };
environment.systemPackages = [ pkgs.kubernetes-helm ]; environment.systemPackages = [ pkgs.kubernetes-helm ];
}; };
kubeConfig = {
services.kubernetes = {
verbose = false;
caFile = ca_pem;
flannel.enable = true;
clusterCidr = "10.10.0.0/16";
etcd = {
servers = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
proxy = {
inherit kubeconfig;
};
};
};
baseConfig = node: { baseConfig = node: {
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ]; imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking = { networking = {
@@ -130,7 +136,7 @@ let
}; };
services.dnsmasq.enable = true; services.dnsmasq.enable = true;
services.dnsmasq.servers = [ services.dnsmasq.servers = [
"/cluster.local/10.253.18.100#4053" "/cluster.local/10.0.0.254#53"
]; ];
}; };
@@ -142,7 +148,7 @@ let
{ {
deployment.targetHost = ip; deployment.targetHost = ip;
require = [ base kubeConfig kubeNode ]; require = [ base kubeConfig kubeNode ];
services.kubernetes.dns.enable = false; services.kubernetes.addons.dns.enable = false;
}; };
in in
{ {
@@ -172,7 +178,7 @@ in
{ {
deployment.targetHost = "10.253.18.101"; deployment.targetHost = "10.253.18.101";
require = [ base etcd kubeConfig kubeNode ]; require = [ base etcd kubeConfig kubeNode ];
services.kubernetes.dns.enable = false; services.kubernetes.addons.dns.enable = false;
}; };
k8s0-2 = { config, lib, pkgs, ... }: k8s0-2 = { config, lib, pkgs, ... }:
@@ -183,6 +189,24 @@ in
{ {
deployment.targetHost = "10.253.18.102"; deployment.targetHost = "10.253.18.102";
require = [ base etcd kubeConfig kubeNode ]; require = [ base etcd kubeConfig kubeNode ];
services.kubernetes.dns.enable = false; services.kubernetes.addons.dns.enable = false;
};
git01 = { config, lib, pkgs, ... }:
let
base = baseConfig "git01";
in
{
deployment.targetHost = "10.253.18.103";
require = [ base kubeConfig kubeNode ];
services.kubernetes.addons.dns.enable = false;
services.nfs.server = {
enable=true;
exports= ''
/vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
networking.firewall.allowedTCPPorts = [ 111 2049 ];
networking.firewall.allowedUDPPorts = [ 111 2049 ];
}; };
} }