oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Major revamp and restructuring of k8s setup:

Browse Source 
* replicated etcd setup for redundancy
* use Flannel network overlay
* naming conventions
* new repo structure
* etc.
This commit is contained in:
Jonas Juselius
2017-07-09 21:04:53 +02:00
parent fb753ab30d
commit b162a82416
23 changed files with 306 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
base/configuration.nix Normal file
View File

@@ -0,0 +1,40 @@
{ config, pkgs, ... }:
{
# Use the GRUB 2 boot loader.
boot.loader.grub.enable = true;
boot.loader.grub.version = 2;
boot.loader.grub.device = "/dev/sda";
boot.kernel.sysctl."vm.overcommit_memory"= 1;
services.vmwareGuest.enable = true;
# Select internationalisation properties.
i18n = {
consoleFont = "Lat2-Terminus16";
consoleKeyMap = "us";
defaultLocale = "en_US.UTF-8";
};
# Set your time zone.
time.timeZone = "Europe/Oslo";
networking.search = [ "itpartner.intern" "itpartner.no" ];
# Enable the OpenSSH daemon.
services.openssh.enable = true;
# The NixOS release to be compatible with for stateful data such as databases.
system.stateVersion = "17.03";
programs.zsh.enable = true;
programs.tmux.enable = true;
virtualisation.docker.enable = true;
security.rtkit.enable = true;
fileSystems."/data" = {
device = "10.253.18.103:/data";
fsType = "nfs";
};
imports = [ ./users.nix ./packages.nix ];
}

0
share/nix-home.nix → base/nix-home.nix
View File

64
base/packages.nix Normal file
View File

@@ -0,0 +1,64 @@
{ config, pkgs, ... }:
let
nix-home = pkgs.callPackage ./nix-home.nix {};
in
{
nixpkgs.config.allowUnfree = true;
environment.systemPackages =
with pkgs;
let
sys = [
stdenv
findutils
coreutils
psmisc
iputils
nettools
netcat
rsync
htop
iotop
zsh
wget
vimNox
python
file
bc
sshuttle
nix-prefetch-git
docker
nix-home
];
devel = [
git
patchelf
binutils
gcc
];
dotnet = with dotnetPackages; [
fsharp
mono-addins
mono
Fake
Paket
];
node = with nodePackages; [
nodejs
npm
npm2nix
yo
gulp
];
k8s = [
ebtables
ethtool
socat
];
in
devel ++
dotnet ++
node ++
k8s ++
sys;
}

13
share/users.nix → base/users.nix
View File

@@ -26,4 +26,17 @@
"ssh-dss AAAAB3NzaC1kc3MAAACBANmiPELldjlhW4SKi9NVVN8DIpRouRj2j/v0ycySHYJv0lCE2ATggXMWY/T25eqMTEtwT7U+7g0MIHxR+GLFLpp6N7CiKh3pS0nj4Ig+f9hX2PF5HR6fOgQVCWcNrQTOV6jjZqWjbZpKYDpVfwHZxWrhgAv+9I+w0MHICulDca5RAAAAFQCjmmSZGG137bgGuLPXZkyvc8DXswAAAIB1bMTmPWS1qsZ5H4hgzoKcW+5b+yD7Yn62GFmZS/n4RdyJt7gBJwxukXaTs9B5g922lem4Tk6W6kslCzAu6Y7JDOkhX/hWasb6fGdCmmK/btqwi2imGeVJImAoFoTKfm4JprKcOmSATGMgTlzFHYFDpngyZ9pFnyubI829zfzNTwAAAIEAhwQwqEhBOT8cEKZRiDExi7jBk7zRKYhX1Wb6uUKI07qQFTLehUIahirHxqXcDhlcxzgHcwXKt6CBPYvre9qhqP6865Be/KecYycntVsx/o77Hv5bqETXojhLhb8I3hD1UnxIJ1FXVOnhL+SbO46oICsghBApDtgTX9iRFnuw0fU= jonas"
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvIIQoi0aM2XmX7evnte/sQTCczSYYg0O0KT7g6Xao4nAoiMZ7udxOijd0vD8VBRSVuz6epcuPLsD0z6skiCJFpT0gG0KkTS6dLZMD3+KOvmvolpuGvRv6Ad/bO05YUvGHJAPdwwCxcXajtBoHOd+KUq8xBqyexgi20i+4P/JulY+RQKPlnQHlb6glcDAjt9RPh96t9T5lCoMAqMWtzV9GZE8/H+o6nMf9pxTjxT/oW/8EKZVgDgCSnpZg668Xj0UNcJW/ba3kSpjUsrdvZgM1E3TVgJ/YZDpM01m9hHS7PBcsJu6RyUMhamAVlsYS4Vy2ylU5rYAPTh23CZh24KD3+AtzTd9vhDNLz+KNKDzgW5b3IHoOyXG78RtPw7gHBGlGQc9OlJqTSLZAFWE2PNP1Pa6q0mqWcJuakSwyQoHvS+8PgEBr0eKKebwuXauuft1DvMNpi9SFxJY5Oy7ck62WzbBDNFDOTdNabXUV/QdOv5Zc2XrUCkr6C1dk57C0G8KTJCkqZIV1XMYZIlDZPuqlK8jZdPjRAnRmkYvvitb0FsvixpsWll1PRj9bYYZ01xX6djuDRvc1FeRG199HKOVWN53l9EMTpy0coxl2hZpbnyfxLdS/tkPkoSuWeY/MKWrn4yd1Q2+edjhTvnn6o6kfnKXKBkuoQvdaX/nyig+X+Q== august.s.solvang@gmail.com"
];
security.sudo.wheelNeedsPassword = false;
security.sudo.extraConfig =
''
Defaults:root,%wheel env_keep+=LOCALE_ARCHIVE
Defaults:root,%wheel env_keep+=NIX_PATH
Defaults:root,%wheel env_keep+=TERMINFO_DIRS
Defaults env_keep+=SSH_AUTH_SOCK
Defaults lecture=never
Defaults shell_noargs
root ALL=(ALL) SETENV: ALL
%wheel ALL=(ALL) NOPASSWD: ALL, SETENV: ALL
'';
}

0
share/workstation.nix → base/workstation.nix
View File

14
git.nix
View File

@@ -2,6 +2,18 @@
git01 = { config, lib, pkgs, ... }:
{
deployment.targetHost = "10.253.18.103";
imports = [ ./git01/configuration.nix ];
networking.hostName = "git01"; # Define your hostname
imports = [ ./hw/git01.nix ./git01/configuration.nix ];
services.nfs.server = {
enable=true;
exports= ''
/data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
/data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
/data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
/data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
'';
};
networking.firewall.allowedTCPPorts = [2049 111 20048];
networking.firewall.allowedUDPPorts = [2049 111 20048];
};
}

24
git01/configuration.nix
View File

@@ -1,24 +0,0 @@
{ config, pkgs, ... }:
{
imports = [
./hardware-configuration.nix
../share/configuration.nix
../share/users.nix
];
networking.hostName = "git01"; # Define your hostname
networking.firewall.allowedTCPPorts = [2049 111 20048];
networking.firewall.allowedUDPPorts = [2049 111 20048];
boot.loader.grub.device = "/dev/sda";
services.nfs.server = {
enable=true;
exports= ''
/data 10.253.18.104(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
/data 10.253.18.100(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
/data 10.253.18.102(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
/data 10.253.18.101(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0)
'';
};
}

0
git01/hardware-configuration.nix → hw/git01.nix
View File

0
k8s0-0/hardware-configuration.nix → hw/k8s0-0.nix
View File

0
k8s0-1/hardware-configuration.nix → hw/k8s0-1.nix
View File

0
k8s0-2/hardware-configuration.nix → hw/k8s0-2.nix
View File

128
k8s.nix
View File

@@ -1,19 +1,129 @@
{
k8s00 = { config, lib, pkgs, ... }:
{
deployment.targetHost = "10.253.18.101";
imports = [ ./k8s00/configuration.nix ];
let
etcdConfig = name: {
services.etcd = {
inherit name;
advertiseClientUrls = [ "https://${name}:2379" ];
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
enable = true;
certFile = ./pki/etcd.pem;
keyFile = ./pki/etcd-key.pem;
trustedCaFile = ./pki/ca.pem;
peerClientCertAuth = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
initialCluster = [
"etcd0=https://etcd0:2380"
"etcd1=https://etcd1:2380"
];
# environment.variables = {
# ETCDCTL_CERT_FILE = ./pki/client.pem;
# ETCDCTL_KEY_FILE = ./pki/client-key.pem;
# ETCDCTL_CA_FILE = ./pki/ca.pem;
# ETCDCTL_PEERS = "https://127.0.0.1:2379";
# };
};
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
};
k8s01 = { config, lib, pkgs, ... }:
flannelConfig = {
services.flannel = {
enable = true;
network = "10.10.0.0/16";
iface = "enp0s3";
etcd = {
endpoints = ["https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
caFile = ./pki/ca.pem;
};
};
};
kubeNode = {
services.kubernetes = {
# verbose = true;
roles = [ "node" ];
kubeconfig = {
server = "https://kubernetes:443";
caFile = ./pki/ca.pem;
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
};
etcd = {
servers = [ "https://etcd0:2379" "https://etcd1:2379" ];
certFile = ./pki/client.pem;
keyFile = ./pki/client-key.pem;
caFile = ./pki/ca.pem;
};
# kubelet.clusterDns = "10.10.1.1";
};
networking.firewall.allowedUDPPorts = [ 8472 ]; # VXLAN
networking.firewall.allowedTCPPorts = [ 10250 ];
networking.extraHosts = ''
10.253.18.100 etcd0 kubernetes
10.253.18.101 etcd1
'';
systemd.services.docker.after = [ "flannel.service" ];
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";
};
kubeMaster = {
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
services.kubernetes = {
roles = [ "master" ];
apiserver = {
publicAddress = "0.0.0.0";
address = "0.0.0.0";
clientCaFile = ./pki/ca.pem;
tlsCertFile = ./pki/apiserver.pem;
tlsKeyFile = ./pki/apiserver-key.pem;
kubeletClientCaFile = ./pki/ca.pem;
kubeletClientCertFile = ./pki/client.pem;
kubeletClientKeyFile = ./pki/client-key.pem;
};
scheduler.leaderElect = true;
controllerManager.leaderElect = true;
controllerManager.serviceAccountKeyFile = ./pki/apiserver-key.pem;
};
networking.firewall.allowedTCPPorts = [ 5000 8080 443 53 ];
networking.firewall.allowedUDPPorts = [ 53 ];
systemd.services.flannel.after = [ "etcd.service" ];
};
in
{
k8s0-0 = { config, lib, pkgs, ... }:
let etcd = etcdConfig "etcd0"; in
{
deployment.targetHost = "10.253.18.100";
imports = [ ./k8s01/configuration.nix ];
networking.hostName = "k8s0-0";
imports = [ ./hw/k8s0-0.nix ./base/configuration.nix ];
require = [ etcd flannelConfig ];
# require = [ etcd flannelConfig kubeMaster kubeNode ];
};
k8s02 = { config, lib, pkgs, ... }:
k8s0-1 = { config, lib, pkgs, ... }:
let etcd = etcdConfig "etcd1"; in
{
deployment.targetHost = "10.253.18.101";
networking.hostName = "k8s0-1";
imports = [ ./hw/k8s0-1.nix ./base/configuration.nix ];
require = [ etcd flannelConfig ];
# require = [ etcd flannelConfig kubeNode ];
};
k8s0-2 = { config, lib, pkgs, ... }:
{
deployment.targetHost = "10.253.18.102";
imports = [ ./k8s02/configuration.nix ];
networking.hostName = "k8s0-2";
imports = [ ./hw/k8s0-2.nix ./base/configuration.nix ];
require = [ flannelConfig ];
};
}

35
k8s0-0/configuration.nix
View File

@@ -1,35 +0,0 @@
{ config, pkgs, ... }:
{
imports = [
./hardware-configuration.nix
../share/configuration.nix
../share/users.nix
../share/k8s.nix
];
networking.hostName = "kubemaster"; # Define your hostname.
networking.firewall.allowedTCPPorts = [ 5000 8080 443 ];
boot.loader.grub.device = "/dev/sda";
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
};
services.kubernetes.roles = [ "master" ];
services.kubernetes.apiserver = {
publicAddress = "0.0.0.0";
address = "0.0.0.0";
# kubeletClientCaFile = ../pki/ca.pem;
# kubeletClientCertFile = ../pki/client.pem;
# kubeletClientKeyFile = ../pki/client-key.pem;
clientCaFile = ../pki/ca.pem;
tlsCertFile = ../pki/apiserver.pem;
tlsKeyFile = ../pki/apiserver-key.pem;
};
services.kubernetes.kubeconfig = {
certFile = ../pki/kubemaster.pem;
keyFile = ../pki/kubemaster-key.pem;
};
}

18
k8s0-1/configuration.nix
View File

@@ -1,18 +0,0 @@
{ config, pkgs, ... }:
{
imports = [
./hardware-configuration.nix
../share/configuration.nix
../share/users.nix
../share/k8s.nix
];
boot.loader.grub.device = "/dev/sda";
networking.hostName = "kub01"; # Define your hostname.
services.kubernetes.kubeconfig = {
certFile = ../pki/kub01.pem;
keyFile = ../pki/kub01-key.pem;
};
}

17
k8s0-2/configuration.nix
View File

@@ -1,17 +0,0 @@
{ config, pkgs, ... }:
{
imports = [
./hardware-configuration.nix
../share/configuration.nix
../share/users.nix
../share/k8s.nix
];
boot.loader.grub.device = "/dev/sda";
networking.hostName = "kub02"; # Define your hostname.
services.kubernetes.kubeconfig = {
certFile = ../pki/kub02.pem;
keyFile = ../pki/kub02-key.pem;
};
}

4
pki/apiserver.json
View File

@@ -1,7 +1,7 @@
{
"hosts": [
"kubemaster",
"10.253.18.101"
"k8s0-0",
"10.253.18.100"
],
"key": {
"algo": "rsa",

1
pki/ca.json
View File

@@ -1,5 +1,6 @@
{
"hosts": [
"itpartner.no",
"itpartner.intern",
"cluster.local"
],

5
pki/client.json
View File

@@ -1,9 +1,4 @@
{
"CN": "@host@",
"hosts": [
"@host@",
"@ip@"
],
"key": {
"algo": "rsa",
"size": 2048

19
pki/etcd.json Normal file
View File

@@ -0,0 +1,19 @@
{
"hosts": [
"etcd0",
"etcd1"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "NO",
"L": "Tromsø",
"O": "Serit IT Partner Tromsø AS",
"OU": "",
"ST": ""
}
]
}

50
pki/mkcerts.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
hosts="kubemaster,101 kub01,100 kub02,102"
# hosts="k8s0-0,100 k8s0-1,101 k8s0-2,102"
mkcacert () {
cfssl genkey -initca ca.json | cfssljson -bare ca
@@ -11,37 +11,51 @@ mkapicert () {
| cfssljson -bare apiserver
}
mketcdcert () {
cfssl gencert -ca ca.pem -ca-key ca-key.pem etcd.json \
| cfssljson -bare etcd
}
mkclientcert () {
host=$1
ip=$2
sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
| cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
| cfssljson -bare $host
cfssl gencert -ca ca.pem -ca-key ca-key.pem client.json \
| cfssljson -bare client
}
mkclientcerts () {
for i in $hosts; do
IFS=","
set -- $i
mkclientcert $1 10.253.18.$2
done
}
# mkclientcert () {
# host=$1
# ip=$2
# sed "s/@host@/$host/g; s/@ip@/$ip/g; " client.json \
# | cfssl gencert -ca ca.pem -ca-key ca-key.pem - \
# | cfssljson -bare $host
# }
# mkclientcerts () {
# for i in $hosts; do
# IFS=","
# set -- $i
# mkclientcert $1 10.253.18.$2
# done
# }
case $1 in
all)
mkcacert
mkapicert
mkclientcerts
mketcdcert
mkclientcert
;;
clients)
mkclientcerts
client)
mkclientcert
;;
api)
mkapicert
;;
etcd)
mketcdcert
;;
*)
echo "usege: mkcerts.sh (all|clients|api)"
echo "usege: mkcerts.sh (all|client|api|etcd)"
exit 1
;;
esac

112
share/configuration.nix
View File

@@ -1,112 +0,0 @@
{ config, pkgs, ... }:
let
nix-home = pkgs.callPackage ./nix-home.nix {};
in
{
# Use the GRUB 2 boot loader.
boot.loader.grub.enable = true;
boot.loader.grub.version = 2;
boot.kernel.sysctl."vm.overcommit_memory"= 1;
services.vmwareGuest.enable=true;
# Select internationalisation properties.
i18n = {
consoleFont = "Lat2-Terminus16";
consoleKeyMap = "us";
defaultLocale = "en_US.UTF-8";
};
# Set your time zone.
time.timeZone = "Europe/Oslo";
networking.search = [ "itpartner.intern" ];
# Enable the OpenSSH daemon.
services.openssh.enable = true;
# The NixOS release to be compatible with for stateful data such as databases.
system.stateVersion = "17.09";
programs.zsh.enable = true;
programs.tmux.enable = true;
virtualisation.docker.enable = true;
security.sudo.wheelNeedsPassword = false;
security.sudo.extraConfig =
''
Defaults:root,%wheel env_keep+=LOCALE_ARCHIVE
Defaults:root,%wheel env_keep+=NIX_PATH
Defaults:root,%wheel env_keep+=TERMINFO_DIRS
Defaults env_keep+=SSH_AUTH_SOCK
Defaults lecture=never
Defaults shell_noargs
root ALL=(ALL) SETENV: ALL
%wheel ALL=(ALL) NOPASSWD: ALL, SETENV: ALL
'';
security.rtkit.enable = true;
fileSystems."/data" = {
device = "10.253.18.103:/data";
fsType = "nfs";
};
nixpkgs.config.allowUnfree = true;
environment.systemPackages =
with pkgs;
let
sys = [
stdenv
findutils
coreutils
psmisc
iputils
nettools
netcat
rsync
htop
iotop
zsh
wget
vimNox
python
file
bc
sshuttle
nix-prefetch-git
docker
nix-home
];
devel = [
git
patchelf
binutils
gcc
];
dotnet = with dotnetPackages; [
fsharp
mono-addins
mono
Fake
Paket
];
node = with nodePackages; [
nodejs
npm
npm2nix
yo
gulp
];
k8s = [
ebtables
ethtool
socat
];
in
devel ++
dotnet ++
node ++
k8s ++
sys;
}

22
share/k8s-node.nix
View File

@@ -1,22 +0,0 @@
{ config, pkgs, ... }:
{
services.kubernetes.roles = [ "node" ];
services.kubernetes.kubeconfig = {
server = "https://10.253.18.101:443";
# caFile = ../pki/ca.cert.pem;
# certFile = ../pki/certificate.pem;
# keyFile = ../pki/key.pem;
};
services.kubernetes.apiserver = {
# kubeletClientCaFile = ../pki/ca.cert.pem;
# kubeletClientCertFile = ../pki/certificate.pem;
# kubeletClientKeyFile = ../pki/key.pem;
# clientCaFile = ../pki/ca.cert.pem;
tlsCertFile = ../pki/certificate.pem;
tlsKeyFile = ../pki/key.pem;
};
services.kubernetes.kubelet = {
tlsCertFile = ../pki/certificate.pem;
tlsKeyFile = ../pki/key.pem;
};
}

9
share/k8s.nix
View File

@@ -1,9 +0,0 @@
{ config, pkgs, ... }:
{
services.kubernetes.roles = [ "node" ];
services.kubernetes.kubeconfig = {
server = "https://10.253.18.101:443";
caFile = ../pki/ca.pem;
};
networking.firewall.allowedTCPPorts = [ 10250 ];
}