oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add git01 to k8s cluster pool.

Browse Source
This commit is contained in:
Jonas Juselius
2017-10-26 12:01:52 +02:00
parent cb74a6d72f
commit 336ecccd09
2 changed files with 74 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
certs.nix
View File

@@ -51,7 +51,7 @@ let
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
DNS.4 = k8s0-0.itpartner.no
IP.1 = 10.10.10.1
IP.1 = 10.0.0.1
IP.2 = 10.253.18.100
'';
@@ -65,11 +65,12 @@ let
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = k8s0-0
DNS.2 = k8s0-1
DNS.3 = k8s0-2
DNS.4 = *.itpartner.no
DNS.5 = *.itpartner.intern
DNS.1 = *.itpartner.no
DNS.2 = *.itpartner.intern
DNS.3 = k8s0-0
DNS.4 = k8s0-1
DNS.5 = k8s0-2
DNS.6 = git01
'';
ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";

110
k8s.nix
View File

@@ -2,6 +2,7 @@ with import ./certs.nix;
let
pkgs = import <nixpkgs> {};
kube_apiserver = "https://10.253.18.100:443";
etcdServers = [ "etcd0" "etcd1" "etcd2" ];
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdServers;
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdServers;
@@ -30,55 +31,38 @@ let
systemd.services.flannel.after = [ "etcd.service" ];
};
kubeConfig = {
systemd.services.kubelet.path = [ pkgs.socat ];
services.flannel = {
enable = true;
network = "10.10.0.0/16";
iface = "ens32";
etcd = {
endpoints = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
};
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
systemd.services.docker = {
after = [ "flannel.service" ];
serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
};
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU";
services.kubernetes.etcd = {
servers = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
# services.kubernetes.verbose = true;
kubeconfig = {
caFile = ca_pem;
keyFile = worker_key;
certFile = worker_cert;
server = kube_apiserver;
};
kubeNode = {
services.kubernetes = {
roles = [ "node" ];
kubeconfig = {
server = "https://10.253.18.100:4443";
server = kube_apiserver;
keyFile = worker_key;
certFile = worker_cert;
caFile = ca_pem;
};
kubelet = {
tlsKeyFile = worker_key;
tlsCertFile = worker_cert;
enable = true;
clientCaFile = ca_pem;
tlsKeyFile = worker_key;
tlsCertFile = worker_cert;
networkPlugin = null;
clusterDns = "10.253.18.100";
# clusterDns = "10.253.18.100";
clusterDns = "10.0.0.254";
inherit kubeconfig;
};
};
networking = {
firewall = {
enable = true;
# trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
allowedTCPPorts = [ 53 10250 ];
allowedTCPPorts = [ 53 4194 10250 ];
allowedUDPPorts = [ 53 ];
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
};
@@ -89,11 +73,12 @@ let
kubeMaster = {
services.kubernetes = {
roles = [ "master" ];
kubelet.unschedulable = true;
apiserver = {
address = "0.0.0.0";
publicAddress = "0.0.0.0";
advertiseAddress = "10.253.18.100";
securePort = 4443;
securePort = 443;
tlsKeyFile = apiserver_key;
tlsCertFile = apiserver_cert;
clientCaFile = ca_pem;
@@ -103,19 +88,40 @@ let
serviceAccountKeyFile = apiserver_key;
};
scheduler.leaderElect = true;
controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = apiserver_key;
controllerManager.rootCaFile = ca_pem;
dns.enable = true;
dns.port = 4053;
controllerManager = {
leaderElect = true;
serviceAccountKeyFile = apiserver_key;
rootCaFile = ca_pem;
inherit kubeconfig;
};
addons.dashboard.enable = true;
addons.dns.enable = true;
};
networking.firewall = {
allowedTCPPorts = [ 5000 8080 4443 4053 ];
allowedUDPPorts = [ 4053 ];
allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
# allowedUDPPorts = [ 4053 ];
};
environment.systemPackages = [ pkgs.kubernetes-helm ];
};
kubeConfig = {
services.kubernetes = {
verbose = false;
caFile = ca_pem;
flannel.enable = true;
clusterCidr = "10.10.0.0/16";
etcd = {
servers = etcdEndpoints;
keyFile = etcd_client_key;
certFile = etcd_client_cert;
caFile = ca_pem;
};
proxy = {
inherit kubeconfig;
};
};
};
baseConfig = node: {
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking = {
@@ -130,7 +136,7 @@ let
};
services.dnsmasq.enable = true;
services.dnsmasq.servers = [
"/cluster.local/10.253.18.100#4053"
"/cluster.local/10.0.0.254#53"
];
};
@@ -142,7 +148,7 @@ let
{
deployment.targetHost = ip;
require = [ base kubeConfig kubeNode ];
services.kubernetes.dns.enable = false;
services.kubernetes.addons.dns.enable = false;
};
in
{
@@ -172,7 +178,7 @@ in
{
deployment.targetHost = "10.253.18.101";
require = [ base etcd kubeConfig kubeNode ];
services.kubernetes.dns.enable = false;
services.kubernetes.addons.dns.enable = false;
};
k8s0-2 = { config, lib, pkgs, ... }:
@@ -183,6 +189,24 @@ in
{
deployment.targetHost = "10.253.18.102";
require = [ base etcd kubeConfig kubeNode ];
services.kubernetes.dns.enable = false;
services.kubernetes.addons.dns.enable = false;
};
git01 = { config, lib, pkgs, ... }:
let
base = baseConfig "git01";
in
{
deployment.targetHost = "10.253.18.103";
require = [ base kubeConfig kubeNode ];
services.kubernetes.addons.dns.enable = false;
services.nfs.server = {
enable=true;
exports= ''
/vol 10.253.18.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
networking.firewall.allowedTCPPorts = [ 111 2049 ];
networking.firewall.allowedUDPPorts = [ 111 2049 ];
};
}