oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Migration to new setup

Browse Source
This commit is contained in:
Jonas Juselius
2019-10-15 15:33:43 +02:00
parent 7b59038e50
commit e4765df729
19 changed files with 346 additions and 3212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
lib/certs.nix → lib/certs.nix.old
View File

33
lib/initca.nix Normal file
View File

@@ -0,0 +1,33 @@
with import <nixpkgs> {};
let
initca' =
let
ca_csr = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
key = {
algo = "rsa";
size = 2048;
};
names = [
{
CN = "kubernetes-cluster-ca";
O = "NixOS";
OU = "services.kubernetes.pki.caSpec";
L = "generated";
}
];
});
in
pkgs.runCommand "initca" {
buildInputs = [ pkgs.cfssl ];
} '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \
mkdir -p $out; cp *.pem $out'';
in
# make ca derivation sha depend on initca cfssl output
pkgs.stdenv.mkDerivation {
name = "ca";
src = initca';
buildCommand = ''
mkdir -p $out;
cp -r $src/* $out
'';
}

48
lib/k8s.nix
View File

@@ -1,6 +1,6 @@
{ pkgs, masterNode, etcdNodes, clusterHosts, certs, ...}:
{ pkgs, masterAddress, etcdNodes, clusterHosts, certs, ...}:
let
kubeApiserver = "https://${masterNode}:8443";
kubeApiserver = "https://${masterAddress}:8443";
localApiserver = "http://127.0.0.1:8080";
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
@@ -10,8 +10,8 @@ rec {
services.etcd = {
inherit name;
enable = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
listenClientUrls = [ "https://0.0.0.0:2379" ];
listenPeerUrls = [ "https://0.0.0.0:2380" ];
peerClientCertAuth = true;
keyFile = certs.etcd.key;
certFile = certs.etcd.cert;
@@ -30,7 +30,7 @@ rec {
systemd.services.flannel.after = [ "etcd.service" ];
};
clientConf = instance: {
clientConf = instance: {
server = kubeApiserver;
keyFile = certs.${instance}.key;
certFile = certs.${instance}.cert;
@@ -39,6 +39,7 @@ rec {
kubeNode = instance: {
services.kubernetes = rec {
inherit masterAddress;
roles = [ "node" ];
kubeconfig = clientConf instance;
kubelet = {
@@ -69,8 +70,8 @@ rec {
roles = [ "master" ];
kubelet.unschedulable = false;
apiserver = {
bindAddress = "0.0.0.0"; #masterNode;
advertiseAddress = masterNode;
bindAddress = "0.0.0.0"; #masterAddress;
advertiseAddress = masterAddress;
authorizationMode = [ "Node" "RBAC" ];
securePort = 8443;
tlsKeyFile = certs.apiserver.key;
@@ -80,12 +81,18 @@ rec {
kubeletClientKeyFile = certs.apiserver.key;
kubeletClientCertFile = certs.apiserver.cert;
serviceAccountKeyFile = certs.apiserver.key;
etcd = {
servers = etcdEndpoints;
keyFile = certs.apiserver.key;
certFile = certs.apiserver.cert;
caFile = certs.ca.cert;
};
};
scheduler.leaderElect = true;
controllerManager = {
leaderElect = true;
serviceAccountKeyFile = certs.apiserver.key;
rootCaFile = certs.ca.cert;
# rootCaFile = certs.ca.cert;
kubeconfig.server = localApiserver;
};
scheduler.kubeconfig.server = localApiserver;
@@ -96,7 +103,7 @@ rec {
version = "v1.10.0";
rbac.enable = true;
rbac.clusterAdmin = true;
tokenTtl = 0;
# tokenTtl = 0;
image = {
imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
imageDigest = "sha256:1d2e1229a918f4bc38b5a3f9f5f11302b3e71f8397b492afac7f273a0008776a";
@@ -114,16 +121,9 @@ rec {
kubeConfig = instance: {
services.kubernetes = {
verbose = false;
caFile = certs.ca.cert;
# caFile = certs.ca.cert;
flannel.enable = true;
clusterCidr = "10.10.0.0/16";
etcd = {
servers = etcdEndpoints;
keyFile = certs.apiserver.key;
certFile = certs.apiserver.cert;
caFile = certs.ca.cert;
};
proxy = {
kubeconfig = clientConf "kube-proxy";
};
@@ -149,9 +149,9 @@ rec {
networking = {
hostName = instance;
extraHosts = clusterHosts;
# nameservers = [ masterNode ];
# nameservers = [ masterAddress ];
# dhcpcd.extraConfig = ''
# static domain_name_servers=${masterNode}
# static domain_name_servers=${masterAddress}
# '';
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
firewall.allowedTCPPorts = [ 80 443 111 ];
@@ -200,15 +200,5 @@ rec {
(kubeConfig name)
(kubeNode name)
];
services.dockerRegistry = {
enable = true;
listenAddress = "0.0.0.0";
enableDelete = true;
enableGarbageCollect = true;
extraConfig = {
REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}";
REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}";
};
};
};
}

142
lib/pki.nix
View File

@@ -1,4 +1,5 @@
{ pkgs ? import <nixpkgs> {} }: rec {
{ pkgs ? import <nixpkgs> {} }:
let
ca-config = pkgs.writeText "ca-config.json" ''
{
"signing": {
@@ -38,20 +39,28 @@
initca' =
let
ca_csr = gencsr {
name = "kubernetes";
cn = "kubernetes";
o = "kubernetes";
hosts = "";
};
ca_csr = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
key = {
algo = "rsa";
size = 2048;
};
names = [
{
CN = "kubernetes-cluster-ca";
O = "NixOS";
OU = "services.kubernetes.pki.caSpec";
L = "generated";
}
];
});
in
pkgs.runCommand "initca" {
buildInputs = [ pkgs.cfssl ];
} '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \
mkdir -p $out; cp *.pem $out'';
# make ca derivation sha depend on initca cfssl output
initca = pkgs.stdenv.mkDerivation {
# make ca derivation sha depend on initca cfssl output
initca = pkgs.stdenv.mkDerivation {
name = "ca";
src = initca';
buildCommand = ''
@@ -72,45 +81,16 @@
mkdir -p $out; cp *.pem $out
'';
toSet = cert:
{
key = "${cert}/cert-key.pem";
cert = "${cert}/cert.pem";
};
gencert = conf:
let crt =
pkgs.runCommand "${conf.name}" {
buildInputs = [ pkgs.cfssl ];
} (cfssl conf);
admin = gencert rec {
name = "admin";
csr = gencsr {
inherit name;
cn = "admin";
o = "system:masters";
hosts = "";
};
};
apiserver = hosts:
gencert rec {
name = "kubernetes";
csr = gencsr {
inherit name hosts;
cn = "kubernetes";
o = "kubernetes";
};
};
etcd = hosts: gencert rec {
name = "etcd";
csr = gencsr {
inherit name hosts;
cn = "etcd";
o = "kubernetes";
in
{
key = "${crt}/cert-key.pem";
cert = "${crt}/cert.pem";
};
};
trust = name: hosts:
let
@@ -125,24 +105,68 @@
};
};
kube-proxy = gencert rec {
name = "kube-proxy";
csr = gencsr {
inherit name;
cn = "system:kube-proxy";
o = "system:node-proxier";
hosts = "";
};
};
# certToSet = cert:
# {
# key = "${cert}/cert-key.pem";
# cert = "${cert}/cert.pem";
# };
worker = instance:
gencert rec {
name = instance.name;
# builtins.foldl'
# (a: x: a // { ${x} = (certificates.${x}); })
# { inherit ca; }
# (builtins.attrNames certificates)
in
{
inherit ca;
admin = gencert rec {
name = "admin";
csr = gencsr {
inherit name;
cn = "admin";
o = "system:masters";
hosts = "";
};
};
apiserver = hosts:
gencert rec {
name = "kubernetes";
csr = gencsr {
inherit name hosts;
cn = "kubernetes";
o = "kubernetes";
};
};
etcd = hosts: gencert rec {
name = "etcd";
csr = gencsr {
inherit name hosts;
cn = "etcd";
o = "kubernetes";
};
};
kube-proxy = gencert rec {
name = "kube-proxy";
csr = gencsr {
inherit name;
cn = "system:node:${instance.name}";
o = "system:nodes";
hosts = ''"${instance.name}","${instance.ip}"'';
cn = "system:kube-proxy";
o = "system:node-proxier";
hosts = "";
};
};
};
worker = instance:
gencert rec {
name = instance.name;
csr = gencsr {
inherit name;
cn = "system:node:${instance.name}";
o = "system:nodes";
hosts = ''"${instance.name}","${instance.ip}"'';
};
};
}