WIP: make new cluster config global
This commit is contained in:
Jonas Juselius
246
lib/k8s.nix
246
lib/k8s.nix
@@ -1,114 +1,52 @@
|
||||
{ pkgs, masterAddress, etcdNodes, clusterHosts, certs, ...}:
|
||||
{ pkgs, lib, settings, ...}:
|
||||
with lib;
|
||||
let
|
||||
kubeApiserver = "https://${masterAddress}:8443";
|
||||
localApiserver = "http://127.0.0.1:8080";
|
||||
etcdEndpoints = builtins.map (x: "https://${x}:2379") etcdNodes;
|
||||
etcdCluster = builtins.map (x: "${x}=https://${x}:2380") etcdNodes;
|
||||
cluster-ca = pkgs.stdenv.mkDerivation {
|
||||
name = "cluster-ca";
|
||||
src = ./ca;
|
||||
buildCommand = ''
|
||||
mkdir -p $out
|
||||
cp $src/* $out
|
||||
'';
|
||||
};
|
||||
nixos-kubernetes-join-nodes = workers:
|
||||
let
|
||||
wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
|
||||
in
|
||||
pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
|
||||
#!/bin/sh
|
||||
set -e
|
||||
token=$(cat /var/lib/cfssl/apitoken.secret)
|
||||
for i in ${wrk}; do
|
||||
ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
|
||||
done
|
||||
'';
|
||||
cidr = "10.10.0.0/16";
|
||||
in
|
||||
rec {
|
||||
etcdConfig = name: {
|
||||
services.etcd = {
|
||||
inherit name;
|
||||
enable = true;
|
||||
listenClientUrls = [ "https://0.0.0.0:2379" ];
|
||||
listenPeerUrls = [ "https://0.0.0.0:2380" ];
|
||||
peerClientCertAuth = true;
|
||||
keyFile = certs.etcd.key;
|
||||
certFile = certs.etcd.cert;
|
||||
trustedCaFile = certs.ca.cert;
|
||||
advertiseClientUrls = [ "https://${name}:2379" ];
|
||||
initialAdvertisePeerUrls = [ "https://${name}:2380" ];
|
||||
initialCluster = etcdCluster;
|
||||
};
|
||||
environment.variables = {
|
||||
ETCDCTL_KEY_FILE = "${certs.admin.key}";
|
||||
ETCDCTL_CERT_FILE = "${certs.admin.cert}";
|
||||
ETCDCTL_CA_FILE = "${certs.ca.cert}";
|
||||
ETCDCTL_PEERS = "https://127.0.0.1:2379";
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 2379 2380 ];
|
||||
systemd.services.flannel.after = [ "etcd.service" ];
|
||||
};
|
||||
|
||||
clientConf = instance: {
|
||||
server = kubeApiserver;
|
||||
keyFile = certs.${instance}.key;
|
||||
certFile = certs.${instance}.cert;
|
||||
caFile = certs.ca.cert;
|
||||
};
|
||||
|
||||
kubeNode = instance: {
|
||||
services.kubernetes = rec {
|
||||
inherit masterAddress;
|
||||
roles = [ "node" ];
|
||||
kubeconfig = clientConf instance;
|
||||
kubelet = {
|
||||
enable = true;
|
||||
clientCaFile = certs.ca.cert;
|
||||
tlsKeyFile = certs.${instance}.key;
|
||||
tlsCertFile = certs.${instance}.cert;
|
||||
networkPlugin = null;
|
||||
clusterDns = "10.0.0.254";
|
||||
extraOpts = "--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice";
|
||||
inherit kubeconfig;
|
||||
};
|
||||
};
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [ 4194 10250 ];
|
||||
# allowedUDPPorts = [ 53 ];
|
||||
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
||||
};
|
||||
};
|
||||
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
||||
virtualisation.docker.autoPrune.enable = true;
|
||||
};
|
||||
|
||||
kubeMaster = {
|
||||
services.cfssl.ca = "${cluster-ca}/ca.pem";
|
||||
services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
|
||||
services.kubernetes = {
|
||||
roles = [ "master" ];
|
||||
masterAddress = settings.master;
|
||||
apiserverAddress = settings.apiserverAddress;
|
||||
clusterCidr = cidr;
|
||||
kubelet.unschedulable = false;
|
||||
pki.genCfsslCACert = false;
|
||||
pki.caCertPathPrefix = "${cluster-ca}/ca";
|
||||
apiserver = {
|
||||
bindAddress = "0.0.0.0"; #masterAddress;
|
||||
advertiseAddress = masterAddress;
|
||||
advertiseAddress = settings.masterAddress;
|
||||
authorizationMode = [ "Node" "RBAC" ];
|
||||
securePort = 8443;
|
||||
tlsKeyFile = certs.apiserver.key;
|
||||
tlsCertFile = certs.apiserver.cert;
|
||||
clientCaFile = certs.ca.cert;
|
||||
kubeletClientCaFile = certs.ca.cert;
|
||||
kubeletClientKeyFile = certs.apiserver.key;
|
||||
kubeletClientCertFile = certs.apiserver.cert;
|
||||
serviceAccountKeyFile = certs.apiserver.key;
|
||||
etcd = {
|
||||
servers = etcdEndpoints;
|
||||
keyFile = certs.apiserver.key;
|
||||
certFile = certs.apiserver.cert;
|
||||
caFile = certs.ca.cert;
|
||||
};
|
||||
insecurePort = 8080;
|
||||
extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
|
||||
};
|
||||
scheduler.leaderElect = true;
|
||||
controllerManager = {
|
||||
leaderElect = true;
|
||||
serviceAccountKeyFile = certs.apiserver.key;
|
||||
# rootCaFile = certs.ca.cert;
|
||||
kubeconfig.server = localApiserver;
|
||||
};
|
||||
scheduler.kubeconfig.server = localApiserver;
|
||||
addons.dns.enable = true;
|
||||
addons.dns.reconcileMode = "EnsureExists";
|
||||
addons.dashboard = rec {
|
||||
enable = true;
|
||||
version = "v1.10.0";
|
||||
rbac.enable = true;
|
||||
rbac.clusterAdmin = true;
|
||||
# tokenTtl = 0;
|
||||
image = {
|
||||
imageName = "k8s.gcr.io/kubernetes-dashboard-amd64";
|
||||
imageDigest = "sha256:1d2e1229a918f4bc38b5a3f9f5f11302b3e71f8397b492afac7f273a0008776a";
|
||||
finalImageTag = version;
|
||||
sha256 = "10qkqqhzkr0bcv0dlf8nq069h190pw6zjj1l5s5g438g80v8639j";
|
||||
addons = {
|
||||
dns = {
|
||||
enable = true;
|
||||
# clusterDomain = "local";
|
||||
reconcileMode = "EnsureExists";
|
||||
};
|
||||
};
|
||||
};
|
||||
@@ -116,39 +54,50 @@ rec {
|
||||
allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
|
||||
allowedUDPPorts = [ 53 4053 ];
|
||||
};
|
||||
environment.systemPackages = [ pkgs.kubernetes-helm ];
|
||||
environment.systemPackages = [
|
||||
pkgs.kubernetes-helm
|
||||
(nixos-kubernetes-join-nodes settings.workers)
|
||||
];
|
||||
};
|
||||
|
||||
kubeConfig = instance: {
|
||||
services.kubernetes = {
|
||||
# caFile = certs.ca.cert;
|
||||
flannel.enable = true;
|
||||
clusterCidr = "10.10.0.0/16";
|
||||
proxy = {
|
||||
kubeconfig = clientConf "kube-proxy";
|
||||
kubeWorker = {
|
||||
services.kubernetes = rec {
|
||||
roles = [ "node" ];
|
||||
clusterCidr = cidr;
|
||||
masterAddress = settings.master;
|
||||
apiserverAddress = settings.apiserverAddress;
|
||||
};
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [ 4194 10250 ];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
||||
};
|
||||
};
|
||||
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
||||
virtualisation.docker.autoPrune.enable = true;
|
||||
};
|
||||
|
||||
nixosConfig = instance: {
|
||||
baseNixos = name: {
|
||||
imports = [
|
||||
(../nixos/hardware-configuration + "/${instance}.nix")
|
||||
(../nixos/hardware-configuration + "/${name}.nix")
|
||||
../nixos/configuration.nix
|
||||
];
|
||||
security.pki.certificateFiles = [
|
||||
certs.ca.cert
|
||||
"${cluster-ca}/ca.pem"
|
||||
];
|
||||
services.glusterfs = {
|
||||
enable = true;
|
||||
tlsSettings = {
|
||||
caCert = certs.ca.cert;
|
||||
tlsKeyPath = certs.${instance}.key;
|
||||
tlsPem = certs.${instance}.cert;
|
||||
};
|
||||
};
|
||||
# services.glusterfs = {
|
||||
# enable = true;
|
||||
# # tlsSettings = {
|
||||
# # caCert = certs.ca.caFile;
|
||||
# # tlsKeyPath = certs.self.keyFile;
|
||||
# # tlsPem = certs.self.certFile;
|
||||
# };
|
||||
# };
|
||||
networking = {
|
||||
hostName = instance;
|
||||
extraHosts = clusterHosts;
|
||||
hostName = name;
|
||||
extraHosts = settings.clusterHosts;
|
||||
# nameservers = [ masterAddress ];
|
||||
# dhcpcd.extraConfig = ''
|
||||
# static domain_name_servers=${masterAddress}
|
||||
@@ -159,46 +108,29 @@ rec {
|
||||
};
|
||||
};
|
||||
|
||||
plain = ip: name: { config, lib, pkgs, ... }:
|
||||
apiserver = ip: name: self:
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [
|
||||
(nixosConfig name)
|
||||
];
|
||||
};
|
||||
|
||||
worker = ip: name: { config, lib, pkgs, ... }:
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [
|
||||
(nixosConfig name)
|
||||
(kubeConfig name)
|
||||
(kubeNode name)
|
||||
];
|
||||
services.kubernetes.addons.dns.enable = false;
|
||||
};
|
||||
|
||||
server = ip: name: etc: { config, lib, pkgs, ... }:
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [
|
||||
(nixosConfig name)
|
||||
(etcdConfig etc)
|
||||
(kubeConfig name)
|
||||
(kubeNode name)
|
||||
];
|
||||
services.kubernetes.addons.dns.enable = false;
|
||||
};
|
||||
|
||||
apiserver = ip: name: etc: { config, lib, pkgs, ... }:
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [
|
||||
(nixosConfig name)
|
||||
(etcdConfig etc)
|
||||
(baseNixos name)
|
||||
kubeMaster
|
||||
(kubeConfig name)
|
||||
(kubeNode name)
|
||||
];
|
||||
};
|
||||
|
||||
worker = ip: name: self:
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [
|
||||
(baseNixos name)
|
||||
kubeWorker
|
||||
];
|
||||
};
|
||||
|
||||
host = ip: name: self:
|
||||
{
|
||||
deployment.targetHost = ip;
|
||||
require = [
|
||||
(baseNixos name)
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user