oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Lots of small fixes. Now it works!

Browse Source
This commit is contained in:
Jonas Juselius
2017-07-10 13:40:09 +02:00
parent bacb4ff2dd
commit c21abb5a5f
5 changed files with 98 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
k8s.nix
View File

@@ -2,80 +2,86 @@ let
etcdConfig = name: { etcdConfig = name: {
services.etcd = { services.etcd = {
inherit name; inherit name;
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
enable = true; enable = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
peerClientCertAuth = true;
certFile = ./pki/etcd.pem; certFile = ./pki/etcd.pem;
keyFile = ./pki/etcd-key.pem; keyFile = ./pki/etcd-key.pem;
trustedCaFile = ./pki/ca.pem; trustedCaFile = ./pki/ca.pem;
peerClientCertAuth = true; advertiseClientUrls = [ "https://${name}:2379" ];
listenClientUrls = ["https://0.0.0.0:2379"]; initialAdvertisePeerUrls = [ "https://${name}:2380" ];
listenPeerUrls = ["https://0.0.0.0:2380"];
initialCluster = [ initialCluster = [
"etcd0=https://etcd0:2380" "etcd0=https://etcd0:2380"
"etcd1=https://etcd1:2380" "etcd1=https://etcd1:2380"
]; ];
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki/client.pem;
# ETCDCTL_KEY_FILE = ./pki/client-key.pem;
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
}; };
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki + "/${name}.pem";
# ETCDCTL_KEY_FILE = ./pki + "/${name}-key.pem";
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
networking.firewall.allowedTCPPorts = [ 2379 2380 ]; networking.firewall.allowedTCPPorts = [ 2379 2380 ];
}; };
flannelConfig = { flannelConfig = node: {
services.flannel = { services.flannel = {
enable = true; enable = true;
network = "10.10.0.0/16"; network = "10.10.0.0/16";
iface = "enp0s3"; iface = "enp2s0";
etcd = { etcd = {
endpoints = ["https://etcd0:2379" "https://etcd1:2379" ]; endpoints = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem; certFile = ./pki + "/${node}.pem";
keyFile = ./pki/client-key.pem; keyFile = ./pki + "/${node}-key.pem";
caFile = ./pki/ca.pem; caFile = ./pki/ca.pem;
}; };
}; };
}; };
kubeNode = { etcdClient = node:{
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki + "/${node}.pem";
keyFile = ./pki + "/${node}-key.pem";
caFile = ./pki/ca.pem;
};
kubeConfig = node: {
require = [ (flannelConfig node) ];
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
# services.kubernetes.verbose = true;
};
kubeNode = doConfig: node: {
require = if doConfig then [ (kubeConfig node) ] else [];
services.kubernetes = { services.kubernetes = {
# verbose = true;
roles = [ "node" ]; roles = [ "node" ];
kubeconfig = { kubeconfig = {
server = "https://kubernetes:443"; server = "https://kubernetes:443";
caFile = ./pki/ca.pem; caFile = ./pki/ca.pem;
certFile = ./pki/client.pem; certFile = ./pki + "/${node}.pem";
keyFile = ./pki/client-key.pem; keyFile = ./pki + "/${node}-key.pem";
}; };
etcd = { kubelet = {
servers = [ "https://etcd0:2379" "https://etcd1:2379" ]; tlsCertFile = ./pki + "/${node}.pem";
certFile = ./pki/client.pem; tlsKeyFile = ./pki + "/${node}-key.pem";
keyFile = ./pki/client-key.pem; networkPlugin = null;
caFile = ./pki/ca.pem; clusterDns = "10.253.18.100";
}; };
# kubelet.clusterDns = "10.10.1.1"; etcd = if doConfig then (etcdClient node) else {};
}; };
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
}; };
kubeMaster = { kubeMaster = node: {
require = [ (kubeConfig node) (kubeNode false node)];
services.dockerRegistry = { services.dockerRegistry = {
enable = true; enable = true;
listenAddress = "0.0.0.0"; listenAddress = "0.0.0.0";
}; };
services.kubernetes = { services.kubernetes = {
roles = [ "master" ]; roles = [ "master" ];
apiserver = { apiserver = {
@@ -84,10 +90,11 @@ let
clientCaFile = ./pki/ca.pem; clientCaFile = ./pki/ca.pem;
tlsCertFile = ./pki/apiserver.pem; tlsCertFile = ./pki/apiserver.pem;
tlsKeyFile = ./pki/apiserver-key.pem; tlsKeyFile = ./pki/apiserver-key.pem;
kubeletClientCaFile = ./pki/ca.pem; # kubeletClientCaFile = ./pki/ca.pem;
kubeletClientCertFile = ./pki/client.pem; # kubeletClientCertFile = ./pki + "/${node}.pem";
kubeletClientKeyFile = ./pki/client-key.pem; # kubeletClientKeyFile = ./pki + "/${node}-key.pem";
}; };
etcd = (etcdClient node);
scheduler.leaderElect = true; scheduler.leaderElect = true;
controllerManager.leaderElect = true; controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem; controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
@@ -98,9 +105,13 @@ let
systemd.services.flannel.after = [ "etcd.service" ]; systemd.services.flannel.after = [ "etcd.service" ];
}; };
baseConfig = name: { baseConfig = node: {
networking.hostName = name; networking.hostName = node;
imports = [ "./hw/${name}.nix" ./base/configuration.nix ]; imports = [ (./hw + "/${node}.nix") ./base/configuration.nix ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
}; };
in in
{ {
@@ -108,31 +119,31 @@ in
let let
etcd = etcdConfig "etcd0"; etcd = etcdConfig "etcd0";
base = baseConfig "k8s0-0"; base = baseConfig "k8s0-0";
master = kubeMaster "k8s0-0";
in in
{ {
deployment.targetHost = "10.253.18.100"; deployment.targetHost = "10.253.18.100";
require = [ base etcd flannelConfig ]; require = [ base etcd master ];
# require = [ base etcd flannelConfig kubeMaster kubeNode ];
}; };
k8s0-1 = { config, lib, pkgs, ... }: k8s0-1 = { config, lib, pkgs, ... }:
let let
etcd = etcdConfig "etcd1"; etcd = etcdConfig "etcd1";
base = baseConfig "k8s0-1"; base = baseConfig "k8s0-1";
node = kubeNode true "k8s0-1";
in in
{ {
deployment.targetHost = "10.253.18.101"; deployment.targetHost = "10.253.18.101";
require = [ base etcd flannelConfig ]; require = [ base etcd node ];
# require = [ base etcd flannelConfig kubeNode ];
}; };
k8s0-2 = { config, lib, pkgs, ... }: k8s0-2 = { config, lib, pkgs, ... }:
let let
base = baseConfig "k8s0-2"; base = baseConfig "k8s0-2";
node = kubeNode true "k8s0-2";
in in
{ {
deployment.targetHost = "10.253.18.102"; deployment.targetHost = "10.253.18.102";
require = [ base flannelConfig ]; require = [ base node ];
# require = [ base flannelConfig kubeNode ];
}; };
} }

1
pki/apiserver.json
View File

@@ -1,6 +1,7 @@
{ {
"hosts": [ "hosts": [
"k8s0-0", "k8s0-0",
"kubernetes",
"10.253.18.100" "10.253.18.100"
], ],
"key": { "key": {

5
pki/client.json
View File

@@ -1,4 +1,9 @@
{ {
"CN": "@host@",
"hosts": [
"@host@",
"@ip@"
],
"key": { "key": {
"algo": "rsa", "algo": "rsa",
"size": 2048 "size": 2048

39
pki/mkcerts.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102" hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102"
mkcacert () { mkcacert () {
cfssl genkey -initca ca.json | cfssljson -bare ca cfssl genkey -initca ca.json | cfssljson -bare ca
@@ -16,27 +16,26 @@ mketcdcert () {
| cfssljson -bare etcd | cfssljson -bare etcd
} }
# mkclientcert () {
# cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \
# | cfssljson -bare client
# }
mkclientcert () { mkclientcert () {
cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \ host=$1
| cfssljson -bare client ip=$2
sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
| cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
| cfssljson -bare $host
} }
# mkclientcert () { mkclientcerts () {
# host=$1 for i in $hosts; do
# ip=$2 IFS=","
set -- $i
# sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \ mkclientcert $1 10.253.18.$2
# | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \ done
# | cfssljson -bare $host }
# }
# mkclientcerts () {
# for i in $hosts; do
# IFS=","
# set -- $i
# mkclientcert $1 10.253.18.$2
# done
# }
case $1 in case $1 in
all) all)
@@ -46,7 +45,7 @@ case $1 in
mkclientcert mkclientcert
;; ;;
client) client)
mkclientcert mkclientcerts
;; ;;
api) api)
mkapicert mkapicert

11
pki/pki.nix Normal file
View File

@@ -0,0 +1,11 @@
{pkgs, ...}:
let
makeCert = name:
pkgs.runCommand name {
buildInputs = [ pkgs.cfssl ];
} ''cfssl gencert -ca ca.pem -ca-key ca-key.pem ${name}.json \
| cfssljson -bare ${name}'';
in
{
ca_key
}