oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Lots of small fixes. Now it works!

Browse Source
This commit is contained in:
Jonas Juselius
2017-07-10 13:40:09 +02:00
parent bacb4ff2dd
commit c21abb5a5f
5 changed files with 98 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
k8s.nix
View File

@@ -2,80 +2,86 @@ let
etcdConfig = name: {
services.etcd = {
inherit name;
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
enable = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
peerClientCertAuth = true;
certFile = ./pki/etcd.pem;
keyFile = ./pki/etcd-key.pem;
trustedCaFile = ./pki/ca.pem;
peerClientCertAuth = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
initialCluster = [
"etcd0=https://etcd0:2380"
"etcd1=https://etcd1:2380"
];
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki/client.pem;
# ETCDCTL_KEY_FILE = ./pki/client-key.pem;
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
};
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki + "/${name}.pem";
# ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem";
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
};
flannelConfig = {
flannelConfig = node: {
services.flannel = {
enable = true;
network = "10.10.0.0/16";
iface = "enp0s3";
iface = "enp2s0";
etcd = {
endpoints = ["https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
caFile = ./pki/ca.pem;
};
};
};
kubeNode = {
etcdClient = node:{
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
caFile = ./pki/ca.pem;
};
kubeConfig = node: {
require = [ (flannelConfig node) ];
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
# services.kubernetes.verbose = true;
};
kubeNode = doConfig: node: {
require = if doConfig then [ (kubeConfig node) ] else [];
services.kubernetes = {
# verbose = true;
roles = [ "node" ];
kubeconfig = {
server = "https://kubernetes:443";
caFile = ./pki/ca.pem;
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
};
etcd = {
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
caFile = ./pki/ca.pem;
kubelet = {
tlsCertFile = ./pki + "/${node}.pem";
tlsKeyFile = ./pki + "/${node}-key.pem";
networkPlugin = null;
clusterDns = "10.253.18.100";
};
# kubelet.clusterDns = "10.10.1.1";
etcd = if doConfig then (etcdClient node) else {};
};
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
};
kubeMaster = {
kubeMaster = node: {
require = [ (kubeConfig node) (kubeNode false node)];
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
services.kubernetes = {
roles = [ "master" ];
apiserver = {
@@ -84,10 +90,11 @@ let
clientCaFile = ./pki/ca.pem;
tlsCertFile = ./pki/apiserver.pem;
tlsKeyFile = ./pki/apiserver-key.pem;
kubeletClientCaFile = ./pki/ca.pem;
kubeletClientCertFile = ./pki/client.pem;
kubeletClientKeyFile = ./pki/client-key.pem;
# kubeletClientCaFile = ./pki/ca.pem;
# kubeletClientCertFile = ./pki + "/${node}.pem";
# kubeletClientKeyFile = ./pki + "/${node}-key.pem";
};
etcd = (etcdClient node);
scheduler.leaderElect = true;
controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
@@ -98,9 +105,13 @@ let
systemd.services.flannel.after = [ "etcd.service" ];
};
baseConfig = name: {
networking.hostName = name;
imports = [ "./hw/${name}.nix" ./base/configuration.nix ];
baseConfig = node: {
networking.hostName = node;
imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
};
in
{
@@ -108,31 +119,31 @@ in
let
etcd = etcdConfig "etcd0";
base = baseConfig "k8s0-0";
master = kubeMaster "k8s0-0";
in
{
deployment.targetHost = "10.253.18.100";
require = [ base etcd flannelConfig ];
# require = [ base etcd flannelConfig kubeMaster kubeNode ];
require = [ base etcd master ];
};
k8s0-1 = { config, lib, pkgs, ... }:
let
etcd = etcdConfig "etcd1";
base = baseConfig "k8s0-1";
node = kubeNode true "k8s0-1";
in
{
deployment.targetHost = "10.253.18.101";
require = [ base etcd flannelConfig ];
# require = [ base etcd flannelConfig kubeNode ];
require = [ base etcd node ];
};
k8s0-2 = { config, lib, pkgs, ... }:
let
base = baseConfig "k8s0-2";
node = kubeNode true "k8s0-2";
in
{
deployment.targetHost = "10.253.18.102";
require = [ base flannelConfig ];
# require = [ base flannelConfig kubeNode ];
require = [ base node ];
};
}

1
pki/apiserver.json
View File

@@ -1,6 +1,7 @@
{
"hosts": [
"k8s0-0",
"kubernetes",
"10.253.18.100"
],
"key": {

5
pki/client.json
View File

@@ -1,4 +1,9 @@
{
"CN": "@host@",
"hosts": [
"@host@",
"@ip@"
],
"key": {
"algo": "rsa",
"size": 2048

39
pki/mkcerts.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102"
hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102"
mkcacert () {
cfssl genkey -initca ca.json | cfssljson -bare ca
@@ -16,27 +16,26 @@ mketcdcert () {
| cfssljson -bare etcd
}
# mkclientcert () {
# cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \
# | cfssljson -bare client
# }
mkclientcert () {
cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \
| cfssljson -bare client
host=$1
ip=$2
sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
| cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
| cfssljson -bare $host
}
# mkclientcert () {
# host=$1
# ip=$2
# sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
# | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
# | cfssljson -bare $host
# }
# mkclientcerts () {
# for i in $hosts; do
# IFS=","
# set -- $i
# mkclientcert $1 10.253.18.$2
# done
# }
mkclientcerts () {
for i in $hosts; do
IFS=","
set -- $i
mkclientcert $1 10.253.18.$2
done
}
case $1 in
all)
@@ -46,7 +45,7 @@ case $1 in
mkclientcert
;;
client)
mkclientcert
mkclientcerts
;;
api)
mkapicert

11
pki/pki.nix Normal file
View File

@@ -0,0 +1,11 @@
{pkgs, ...}:
let
makeCert = name:
pkgs.runCommand name {
buildInputs = [ pkgs.cfssl ];
} ''cfssl gencert -ca ca.pem -ca-key ca-key.pem ${name}.json \
| cfssljson -bare ${name}'';
in
{
ca_key
}