oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Modularize k8s configs

Browse Source
This commit is contained in:
Jonas Juselius
2020-10-30 15:08:39 +01:00
parent 521820ceb2
commit b89514eae4
2 changed files with 166 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
lib/default.nix
View File

@@ -11,70 +11,76 @@ let
}; };
}; };
# hostCerts = builtins.foldl' hostCerts = builtins.foldl'
# (a: x: a // { ${x.name} = pki.gencert { (a: x: a // { ${x.name} = pki.gencert {
# cn = x.name; cn = x.name;
# ca = x.ca; ca = x.ca;
# o = cfg.clusterName; o = cfg.clusterName;
# }; };
# }) {} cfg.hosts; }) {} cfg.hosts;
# mkHost = host: self: { mkHost = host: self: {
# deployment.targetHost = host.address; deployment.targetHost = host.address;
# require = [ require = [
# (baseNixos host.name) (baseNixos host.name)
# ]; ];
# }; };
# baseDeployment = nodes: attrs: baseDeployment = nodes: attrs:
# let let
# hosts = hosts =
# builtins.foldl' builtins.foldl'
# (a: x: a // { ${x.name} = mkHost x _; }) {} nodes; (a: x: a // { ${x.name} = mkHost x _; }) {} nodes;
# hosts' = lib.recursiveUpdate hosts attrs; hosts' = lib.recursiveUpdate hosts attrs;
# names = builtins.attrNames hosts; names = builtins.attrNames hosts;
# in in
# builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names; builtins.foldl' (a: x: a // { ${x} = self: hosts'.${x}; }) {} names;
in { in {
k8s = rec {
master = host: self: {
deployment.targetHost = host.address;
# k8s = import ./k8s.nix { inherit pgks lib config; }; cluster = cfg // {
hostName = host.name;
cert = mkCert host.name;
k8s.master.enable = true;
k8s.node.enable = true;
};
# k8s = rec { imports = [ host.hw ./k8s.nix ];
# apiserver = host: self: { };
# deployment.targetHost = host.address;
# require = [
# (os.baseNixos host.name)
# k8s.kubeMaster
# ];
# };
# node = host: self: { node = host: self: {
# deployment.targetHost = host.address; deployment.targetHost = host.address;
# require = [
# (os.baseNixos host.name)
# k8s.kubeWorker
# ];
# };
# deployment = masterNode: workerNodes: cluster = cfg // {
# let hostName = host.name;
# master = { "${master.name}" = apiserver masterNode; }; cert = mkCert host.name;
# in k8s.node.enable = true;
# builtins.foldl' (a: x: };
# a // { "${x.name}" = mkWorker x; }) master workerNodes;
# }; imports = [ host.hw ./k8s.nix ];
};
mkDeployment = masterNode: workerNodes:
let
apiserver = { "${master.name}" = apiserver masterNode; };
in
builtins.foldl' (a: x:
a // { "${x.name}" = mkWorker x; }) apiserver workerNodes;
};
fs = rec { fs = rec {
mkNode = host: self: { mkNode = host: self: {
deployment.targetHost = host.address; deployment.targetHost = host.address;
imports = [ host.hw ./fs.nix ];
cluster = cfg // { cluster = cfg // {
hostName = host.name; hostName = host.name;
cert = mkCert host.name; cert = mkCert host.name;
}; };
imports = [ host.hw ./fs.nix ];
}; };
mkDeployment = nodes: mkDeployment = nodes:
@@ -82,19 +88,23 @@ in {
a // { "${x.name}" = mkNode x; }) {} nodes; a // { "${x.name}" = mkNode x; }) {} nodes;
} ; } ;
# host = rec { host = rec {
# node = host: self: { node = host: self: {
# deployment.targetHost = host.address; deployment.targetHost = host.address;
# require = [
# (os.baseNixos host.name)
# ];
# };
# deployment = masterNode: workerNodes: cluster = cfg // {
# let hostName = host.name;
# master = { "${master.name}" = apiserver masterNode; }; cert = mkCert host.name;
# in };
# builtins.foldl' (a: x:
# a // { "${x.name}" = mkWorker x; }) master workerNodes; imports = [ host.hw ./os.nix ];
# }; };
mkDeployment = masterNode: workerNodes:
let
master = { "${master.name}" = apiserver masterNode; };
in
builtins.foldl' (a: x:
a // { "${x.name}" = mkWorker x; }) master workerNodes;
};
} }

122
lib/k8s.nix
View File

@@ -1,9 +1,13 @@
{ pkgs, lib, settings, here ? ./., ...}: { pkgs, lib, config, ...}:
with import ./base.nix { inherit pkgs lib settings here; };
with lib; with lib;
let let
cfg = config.cluster;
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
apiserverAddress = "https://${masterAddress}:4443"; apiserverAddress = "https://${masterAddress}:4443";
masterAddress = settings.master.address;
masterAddress = cfg.k8s.master.address;
cfssl-apitoken = cfssl-apitoken =
let let
@@ -26,13 +30,13 @@ let
}; };
kube-system-bootstrap = kube-system-bootstrap =
with settings;
let let
worker_nodes = pkgs.writeText "kube-worker-nodes" ( workerNodes = pkgs.writeText "kube-worker-nodes" (
builtins.foldl' (a: x: builtins.foldl' (a: x:
a + " - ${x.address}\n" a + " - ${x.address}\n"
) "" settings.workers); ) "" cfg.k8s.nodes);
grafana_ldap = pkgs.writeText "grafana-ldap.toml" grafana_ldap_toml; grafanaLdap = pkgs.writeText "grafana-ldap.toml"
cfg.k8s.bootstrap.grafana_ldap_toml;
in in
pkgs.stdenv.mkDerivation { pkgs.stdenv.mkDerivation {
name = "kube-system-bootstrap"; name = "kube-system-bootstrap";
@@ -45,16 +49,16 @@ let
mkdir -p $share/charts mkdir -p $share/charts
export bash="${pkgs.bash}" export bash="${pkgs.bash}"
export apiserver="${settings.master.name}" export apiserver="${cfg.k8s.master.name}"
export apiserverAddress="${settings.master.address}" export apiserverAddress="${cfg.k8s.master.address}"
export initca="${pki.initca}" export initca="${pki.initca}"
export cluster="${clusterName}" export cluster="${cfg.clusterName}"
export fileserver="${fileserver}" export fileserver="${cfg.k8s.fileserver}"
export acme_email="${acme_email}" export acme_email="${cfg.k8s.bootrstrap.acme_email}"
export grafana_smtp_user="$(echo -n ${grafana_smtp_user} | base64 -w0)" export grafana_smtp_user="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_user} | base64 -w0)"
export grafana_smtp_password="$(echo -n ${grafana_smtp_password} | base64 -w0)" export grafana_smtp_password="$(echo -n ${cfg.k8s.bootstrap.grafana_smtp_password} | base64 -w0)"
export grafana_ldap_toml="$(cat ${grafana_ldap} | base64 -w0)" export grafana_ldap_toml="$(cat ${grafanaLdap} | base64 -w0)"
export workers="$(cat ${worker_nodes})" export workers="$(cat ${workerNodes})"
substituteAll $src/bin/initial-kube-system-bootstrap $share/bin/initial-kube-system-bootstrap substituteAll $src/bin/initial-kube-system-bootstrap $share/bin/initial-kube-system-bootstrap
chmod 755 $share/bin/initial-kube-system-bootstrap chmod 755 $share/bin/initial-kube-system-bootstrap
@@ -97,14 +101,14 @@ let
services.kubernetes = { services.kubernetes = {
roles = [ "master" ]; roles = [ "master" ];
inherit apiserverAddress; inherit apiserverAddress;
masterAddress = settings.master.name; masterAddress = cfg.k8s.master.name;
clusterCidr = settings.cidr; clusterCidr = cfg.k8s.cidr;
pki.genCfsslCACert = false; pki.genCfsslCACert = false;
pki.genCfsslAPIToken = false; pki.genCfsslAPIToken = false;
pki.caCertPathPrefix = "${pki.initca}/ca"; pki.caCertPathPrefix = "${pki.initca}/ca";
kubelet = { kubelet = {
clusterDomain = "${settings.clusterName}.local"; clusterDomain = "${cfg.clusterName}.local";
}; };
apiserver = { apiserver = {
@@ -114,7 +118,7 @@ let
securePort = 4443; securePort = 4443;
insecurePort = 8080; insecurePort = 8080;
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}"; extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
extraSANs = settings.master.extraSANs; extraSANs = cfg.k8s.master.extraSANs;
# verbosity = 4; # verbosity = 4;
}; };
@@ -129,7 +133,7 @@ let
addons = { addons = {
dns = { dns = {
enable = true; enable = true;
clusterDomain = "${settings.clusterName}.local"; clusterDomain = "${cfg.clusterName}.local";
reconcileMode = "EnsureExists"; reconcileMode = "EnsureExists";
}; };
}; };
@@ -168,13 +172,13 @@ let
}; };
}; };
kubeWorker = { kubeNode = {
services.kubernetes = rec { services.kubernetes = rec {
roles = [ "node" ]; roles = [ "node" ];
inherit apiserverAddress; inherit apiserverAddress;
masterAddress = settings.master.name; masterAddress = cfg.k8s.master.name;
clusterCidr = settings.cidr; clusterCidr = cfg.k8s.cidr;
kubelet.clusterDomain = "${settings.clusterName}.local"; kubelet.clusterDomain = "${cfg.clusterName}.local";
}; };
networking = { networking = {
@@ -199,5 +203,69 @@ let
}; };
}; };
in in {
deployment options.cluster.k8s = {
nodes = mkOption {
type = types.attrs;
default = {};
};
fileserver = mkOption {
type = types.str;
default = "127.0.0.1";
};
cidr = mkOption {
type = types.str;
default = "10.11.0.0/16";
};
master = {
enable = mkEnableOption "Enable kubernetes master node";
address = mkOption {
type = types.str;
default = "127.0.0.1";
};
name = mkOption {
type = types.str;
default = "apiserver";
};
};
node = {
enable = mkEnableOption "Enable kubernetes";
};
bootstrap = {
acme_email = mkOption {
type = types.str;
default = "";
};
grafana_smtp_user = mkOption {
type = types.str;
default = "";
};
grafana_smtp_password = mkOption {
type = types.str;
default = "";
};
grafana_ldap = mkOption {
type = types.str;
default = "";
};
};
};
config = mkMerge [
(mkIf cfg.k8s.master.enable kubeMaster)
(mkIf cfg.k8s.node.enable kubeNode)
];
imports = [ ./os.nix ];
}