Major revamp.
This commit is contained in:
Jonas Juselius
152
lib/certs.nix
Normal file
152
lib/certs.nix
Normal file
@@ -0,0 +1,152 @@
|
||||
let
|
||||
pkgs = import <nixpkgs> {};
|
||||
|
||||
runWithOpenSSL = file: cmd: pkgs.runCommand file {
|
||||
buildInputs = [ pkgs.openssl_1_1_0 ];
|
||||
} ("export RANDFILE=/tmp/rnd;" + cmd);
|
||||
|
||||
etcd_cnf = pkgs.writeText "etcd-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = serverAuth
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = etcd0
|
||||
DNS.2 = etcd1
|
||||
DNS.3 = etcd2
|
||||
DNS.4 = k8s0-0
|
||||
DNS.5 = k8s0-1
|
||||
DNS.6 = k8s0-2
|
||||
IP.1 = 127.0.0.1
|
||||
'';
|
||||
|
||||
etcd_client_cnf = pkgs.writeText "etcd-client-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
extendedKeyUsage = clientAuth
|
||||
'';
|
||||
|
||||
apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = kubernetes
|
||||
DNS.2 = kubernetes.default
|
||||
DNS.3 = kubernetes.default.svc
|
||||
DNS.4 = kubernetes.default.svc.cluster.local
|
||||
DNS.4 = k8s0-0.itpartner.no
|
||||
IP.1 = 10.0.0.1
|
||||
IP.2 = 10.253.18.100
|
||||
'';
|
||||
|
||||
worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
|
||||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = *.itpartner.no
|
||||
DNS.2 = *.itpartner.intern
|
||||
DNS.3 = k8s0-0
|
||||
DNS.4 = k8s0-1
|
||||
DNS.5 = k8s0-2
|
||||
DNS.6 = git01
|
||||
'';
|
||||
|
||||
ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
|
||||
ca_pem = runWithOpenSSL "ca.pem" ''
|
||||
openssl req \
|
||||
-x509 -new -nodes -key ${ca_key} \
|
||||
-days 10000 -out $out -subj "/CN=kube-ca"
|
||||
'';
|
||||
|
||||
etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
|
||||
etcd_csr = runWithOpenSSL "etcd.csr" ''
|
||||
openssl req \
|
||||
-new -key ${etcd_key} \
|
||||
-out $out -subj "/CN=etcd" \
|
||||
-config ${etcd_cnf}
|
||||
'';
|
||||
etcd_cert = runWithOpenSSL "etcd.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${etcd_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} \
|
||||
-CAcreateserial -out $out \
|
||||
-days 365 -extensions v3_req \
|
||||
-extfile ${etcd_cnf}
|
||||
'';
|
||||
|
||||
etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
|
||||
"openssl genrsa -out $out 2048";
|
||||
etcd_client_csr = runWithOpenSSL "etcd-client.csr" ''
|
||||
openssl req \
|
||||
-new -key ${etcd_client_key} \
|
||||
-out $out -subj "/CN=etcd-client" \
|
||||
-config ${etcd_client_cnf}
|
||||
'';
|
||||
etcd_client_cert = runWithOpenSSL "etcd-client.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${etcd_client_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 365 -extensions v3_req \
|
||||
-extfile ${etcd_client_cnf}
|
||||
'';
|
||||
|
||||
apiserver_key = runWithOpenSSL "apiserver-key.pem"
|
||||
"openssl genrsa -out $out 2048";
|
||||
apiserver_csr = runWithOpenSSL "apiserver.csr" ''
|
||||
openssl req \
|
||||
-new -key ${apiserver_key} \
|
||||
-out $out -subj "/CN=kube-apiserver" \
|
||||
-config ${apiserver_cnf}
|
||||
'';
|
||||
apiserver_cert = runWithOpenSSL "apiserver.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${apiserver_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 365 -extensions v3_req \
|
||||
-extfile ${apiserver_cnf}
|
||||
'';
|
||||
|
||||
worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";
|
||||
worker_csr = runWithOpenSSL "worker.csr" ''
|
||||
openssl req \
|
||||
-new -key ${worker_key} \
|
||||
-out $out -subj "/CN=kube-worker" \
|
||||
-config ${worker_cnf}
|
||||
'';
|
||||
worker_cert = runWithOpenSSL "worker.pem" ''
|
||||
openssl x509 \
|
||||
-req -in ${worker_csr} \
|
||||
-CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
|
||||
-out $out -days 365 -extensions v3_req \
|
||||
-extfile ${worker_cnf}
|
||||
'';
|
||||
in
|
||||
{
|
||||
inherit ca_key ca_pem;
|
||||
inherit etcd_key etcd_cert;
|
||||
inherit etcd_client_key etcd_client_cert;
|
||||
inherit apiserver_key apiserver_cert;
|
||||
inherit worker_key worker_cert;
|
||||
}
|
||||
173
lib/k8s.nix
Normal file
173
lib/k8s.nix
Normal file
@@ -0,0 +1,173 @@
|
||||
{ pkgs, kubeMaster, etcdNodes, clusterHosts, certs, ...}:
|
||||
let
|
||||
kubeApiserver = "https://${kubeMaster}:443";
|
||||
localApiserver = "https://127.0.0.1:8080";
|
||||
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
|
||||
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
|
||||
in
|
||||
rec {
|
||||
etcdConfig = name: {
|
||||
services.etcd = {
|
||||
inherit name;
|
||||
enable = true;
|
||||
listenClientUrls = ["https://0.0.0.0:2379"];
|
||||
listenPeerUrls = ["https://0.0.0.0:2380"];
|
||||
peerClientCertAuth = true;
|
||||
keyFile = certs.etcd.key;
|
||||
certFile = certs.etcd.cert;
|
||||
trustedCaFile = certs.ca.cert;
|
||||
advertiseClientUrls = [ "https://${name}:2379" ];
|
||||
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
|
||||
initialCluster = etcdCluster;
|
||||
};
|
||||
environment.variables = {
|
||||
ETCDCTL_KEY_FILE = "${certs.admin.key}";
|
||||
ETCDCTL_CERT_FILE = "${certs.admin.cert}";
|
||||
ETCDCTL_CA_FILE = "${certs.ca.cert}";
|
||||
ETCDCTL_PEERS = "https://127.0.0.1:2379";
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
|
||||
systemd.services.flannel.after = [ "etcd.service" ];
|
||||
};
|
||||
|
||||
clientConf = instance: {
|
||||
server = kubeApiserver;
|
||||
keyFile = certs.${instance}.key;
|
||||
certFile = certs.${instance}.cert;
|
||||
caFile = certs.ca.cert;
|
||||
};
|
||||
|
||||
kubeNode = instance: {
|
||||
services.kubernetes = rec {
|
||||
roles = [ "node" ];
|
||||
kubeconfig = clientConf instance;
|
||||
kubelet = {
|
||||
enable = true;
|
||||
clientCaFile = certs.ca.cert;
|
||||
tlsKeyFile = certs.${instance}.key;
|
||||
tlsCertFile = certs.${instance}.cert;
|
||||
networkPlugin = null;
|
||||
clusterDns = "10.0.0.254";
|
||||
extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice";
|
||||
inherit kubeconfig;
|
||||
};
|
||||
};
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
# trustedInterfaces = [ "flannel.1" "docker0" "veth+" ];
|
||||
allowedTCPPorts = [ 53 4194 10250 ];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
||||
};
|
||||
};
|
||||
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
||||
# systemd.services.kube-proxy.path = [pkgs.iptables pkgs.conntrack_tools pkgs.kmod];
|
||||
};
|
||||
|
||||
kubeMaster = {
|
||||
services.kubernetes = {
|
||||
roles = [ "master" ];
|
||||
kubelet.unschedulable = true;
|
||||
apiserver = {
|
||||
address = kubeMaster;
|
||||
advertiseAddress = kubeMaster;
|
||||
authorizationMode = [ "Node" "RBAC" ];
|
||||
securePort = 443;
|
||||
tlsKeyFile = certs.apiserver.key;
|
||||
tlsCertFile = certs.apiserver.cert;
|
||||
clientCaFile = certs.ca.cert;
|
||||
kubeletClientCaFile = certs.ca.cert;
|
||||
kubeletClientKeyFile = certs.apiserver.key;
|
||||
kubeletClientCertFile = certs.apiserver.cert;
|
||||
serviceAccountKeyFile = certs.apiserver.key;
|
||||
};
|
||||
scheduler.leaderElect = true;
|
||||
controllerManager = {
|
||||
leaderElect = true;
|
||||
serviceAccountKeyFile = certs.apiserver.key;
|
||||
rootCaFile = certs.ca.cert;
|
||||
kubeconfig.server = localApiserver;
|
||||
};
|
||||
scheduler.kubeconfig.server = localApiserver;
|
||||
addons.dashboard.enable = true;
|
||||
addons.dns.enable = true;
|
||||
};
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [ 5000 8080 443 ]; #;4053 ];
|
||||
# allowedUDPPorts = [ 4053 ];
|
||||
};
|
||||
environment.systemPackages = [ pkgs.kubernetes-helm ];
|
||||
};
|
||||
|
||||
kubeConfig = instance: {
|
||||
services.kubernetes = {
|
||||
verbose = false;
|
||||
caFile = certs.ca.cert;
|
||||
flannel.enable = true;
|
||||
clusterCidr = "10.10.0.0/16";
|
||||
etcd = {
|
||||
servers = etcdEndpoints;
|
||||
keyFile = certs.apiserver.key;
|
||||
certFile = certs.apiserver.cert;
|
||||
caFile = certs.ca.cert;
|
||||
};
|
||||
proxy = {
|
||||
kubeconfig = clientConf "kube-proxy";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
nixosConfig = node: {
|
||||
imports = [ (./hardware-configuration + "/${node}.nix") ./nixos/configuration.nix ];
|
||||
networking = {
|
||||
hostName = node;
|
||||
extraHosts = clusterHosts;
|
||||
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
||||
firewall.allowedTCPPorts = [ 80 443 ];
|
||||
};
|
||||
environment.systemPackages = [ pkgs.tshark ];
|
||||
};
|
||||
|
||||
worker = host: ip: { config, lib, pkgs, ... }:
|
||||
let
|
||||
instance = host;
|
||||
base = nixosConfig host;
|
||||
in
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [ base (kubeConfig instance) (kubeNode instance) ];
|
||||
services.kubernetes.addons.dns.enable = false;
|
||||
};
|
||||
|
||||
server = host: etc: ip: { config, lib, pkgs, ... }:
|
||||
let
|
||||
instance = host;
|
||||
base = nixosConfig instance;
|
||||
etcd = etcdConfig etc;
|
||||
in
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [ base etcd (kubeConfig instance) (kubeNode instance) ];
|
||||
services.kubernetes.addons.dns.enable = false;
|
||||
};
|
||||
|
||||
apiserver = host: ip: etc: { config, lib, pkgs, ... }:
|
||||
let
|
||||
instance = host;
|
||||
base = nixosConfig instance;
|
||||
etcd = etcdConfig etc;
|
||||
in
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [ base etcd (kubeConfig instance) kubeMaster (kubeNode instance) ];
|
||||
services.dockerRegistry = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
extraConfig = {
|
||||
REGISTRY_HTTP_TLS_CERTIFICATE = "${certs.apiserver.cert}";
|
||||
REGISTRY_HTTP_TLS_KEY = "${certs.apiserver.key}";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
124
lib/pki.nix
Normal file
124
lib/pki.nix
Normal file
@@ -0,0 +1,124 @@
|
||||
{ pkgs ? import <nixpkgs> {} }: rec {
|
||||
ca-config = pkgs.writeText "ca-config.json" ''
|
||||
{
|
||||
"signing": {
|
||||
"default": {
|
||||
"expiry": "8760h"
|
||||
},
|
||||
"profiles": {
|
||||
"kubernetes": {
|
||||
"usages": [
|
||||
"signing",
|
||||
"key encipherment",
|
||||
"server auth",
|
||||
"client auth"
|
||||
],
|
||||
"expiry": "8760h"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'';
|
||||
|
||||
gencsr = args: pkgs.writeText "${args.name}-csr.json" ''
|
||||
{
|
||||
"CN": "${args.cn}",
|
||||
"hosts": [ ${args.hosts} ],
|
||||
"key": {
|
||||
"algo": "rsa",
|
||||
"size": 2048
|
||||
},
|
||||
"names": [
|
||||
{
|
||||
"O": "${args.o}"
|
||||
}
|
||||
]
|
||||
}
|
||||
'';
|
||||
|
||||
initca =
|
||||
let
|
||||
ca_csr = gencsr {
|
||||
name = "kubernetes";
|
||||
cn = "kubernetes";
|
||||
o = "kubernetes";
|
||||
hosts = "";
|
||||
};
|
||||
in
|
||||
pkgs.runCommand "ca" {
|
||||
buildInputs = [ pkgs.cfssl ];
|
||||
} '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca; \
|
||||
mkdir -p $out; cp *.pem $out'';
|
||||
ca = {
|
||||
key = "${initca}/ca-key.pem";
|
||||
cert = "${initca}/ca.pem";
|
||||
};
|
||||
|
||||
cfssl = conf: ''
|
||||
cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
|
||||
-config=${ca-config} -profile=kubernetes ${conf.csr} | \
|
||||
cfssljson -bare cert; \
|
||||
mkdir -p $out; cp *.pem $out
|
||||
'';
|
||||
|
||||
gencert = conf:
|
||||
let
|
||||
drv = pkgs.runCommand "${conf.name}" {
|
||||
buildInputs = [ pkgs.cfssl ];
|
||||
} (cfssl conf);
|
||||
in
|
||||
{
|
||||
key = "${drv}/cert-key.pem";
|
||||
cert = "${drv}/cert.pem";
|
||||
};
|
||||
|
||||
admin = gencert rec {
|
||||
name = "admin";
|
||||
csr = gencsr {
|
||||
inherit name;
|
||||
cn = "admin";
|
||||
o = "system:masters";
|
||||
hosts = "";
|
||||
};
|
||||
};
|
||||
|
||||
apiserver = hosts:
|
||||
gencert rec {
|
||||
name = "kubernetes";
|
||||
csr = gencsr {
|
||||
inherit name hosts;
|
||||
cn = "kubernetes";
|
||||
o = "kubernetes";
|
||||
};
|
||||
};
|
||||
|
||||
etcd = hosts: gencert rec {
|
||||
name = "etcd";
|
||||
csr = gencsr {
|
||||
inherit name hosts;
|
||||
cn = "etcd";
|
||||
o = "kubernetes";
|
||||
};
|
||||
};
|
||||
|
||||
kube-proxy = gencert rec {
|
||||
name = "kube-proxy";
|
||||
csr = gencsr {
|
||||
inherit name;
|
||||
cn = "system:kube-proxy";
|
||||
o = "system:node-proxier";
|
||||
hosts = "";
|
||||
};
|
||||
};
|
||||
|
||||
worker = instance:
|
||||
gencert rec {
|
||||
name = instance.name;
|
||||
csr = gencsr {
|
||||
inherit name;
|
||||
cn = "system:node:${instance.name}";
|
||||
o = "system:nodes";
|
||||
hosts = ''"${instance.name}","${instance.ip}"'';
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user