137 lines
3.5 KiB
Nix
137 lines
3.5 KiB
Nix
{ pkgs, lib, settings, ...}:
|
|
with lib;
|
|
let
|
|
cluster-ca = pkgs.stdenv.mkDerivation {
|
|
name = "cluster-ca";
|
|
src = ./ca;
|
|
buildCommand = ''
|
|
mkdir -p $out
|
|
cp $src/* $out
|
|
'';
|
|
};
|
|
nixos-kubernetes-join-nodes = workers:
|
|
let
|
|
wrk = builtins.foldl' (a: s: a + " " + s) "" workers;
|
|
in
|
|
pkgs.writeScriptBin "nixos-kubernetes-join-nodes" ''
|
|
#!/bin/sh
|
|
set -e
|
|
token=$(cat /var/lib/cfssl/apitoken.secret)
|
|
for i in ${wrk}; do
|
|
ssh root@$i "echo $token | sh nixos-kubernetes-node-join"
|
|
done
|
|
'';
|
|
cidr = "10.10.0.0/16";
|
|
in
|
|
rec {
|
|
kubeMaster = {
|
|
services.cfssl.ca = "${cluster-ca}/ca.pem";
|
|
services.cfssl.caKey = "${cluster-ca}/ca-key.pem";
|
|
services.kubernetes = {
|
|
roles = [ "master" ];
|
|
masterAddress = settings.master;
|
|
apiserverAddress = settings.apiserverAddress;
|
|
clusterCidr = cidr;
|
|
kubelet.unschedulable = false;
|
|
pki.genCfsslCACert = false;
|
|
pki.caCertPathPrefix = "${cluster-ca}/ca";
|
|
apiserver = {
|
|
advertiseAddress = settings.masterAddress;
|
|
authorizationMode = [ "Node" "RBAC" ];
|
|
securePort = 8443;
|
|
insecurePort = 8080;
|
|
extraOpts = "--requestheader-client-ca-file ${cluster-ca}/ca.pem";
|
|
};
|
|
addons = {
|
|
dns = {
|
|
enable = true;
|
|
# clusterDomain = "local";
|
|
reconcileMode = "EnsureExists";
|
|
};
|
|
};
|
|
};
|
|
networking.firewall = {
|
|
allowedTCPPorts = [ 53 5000 8080 8443 ]; #;4053 ];
|
|
allowedUDPPorts = [ 53 4053 ];
|
|
};
|
|
environment.systemPackages = [
|
|
pkgs.kubernetes-helm
|
|
(nixos-kubernetes-join-nodes settings.workers)
|
|
];
|
|
};
|
|
|
|
kubeWorker = {
|
|
services.kubernetes = rec {
|
|
roles = [ "node" ];
|
|
clusterCidr = cidr;
|
|
masterAddress = settings.master;
|
|
apiserverAddress = settings.apiserverAddress;
|
|
};
|
|
networking = {
|
|
firewall = {
|
|
enable = true;
|
|
allowedTCPPorts = [ 4194 10250 ];
|
|
allowedUDPPorts = [ 53 ];
|
|
extraCommands = ''iptables -m comment --comment "pod external access" -t nat -A POSTROUTING ! -d 10.10.0.0/16 -m addrtype ! --dst-type LOCAL -j MASQUERADE'';
|
|
};
|
|
};
|
|
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
|
virtualisation.docker.autoPrune.enable = true;
|
|
};
|
|
|
|
baseNixos = name: {
|
|
imports = [
|
|
(../nixos/hardware-configuration + "/${name}.nix")
|
|
../nixos/configuration.nix
|
|
];
|
|
security.pki.certificateFiles = [
|
|
"${cluster-ca}/ca.pem"
|
|
];
|
|
# services.glusterfs = {
|
|
# enable = true;
|
|
# # tlsSettings = {
|
|
# # caCert = certs.ca.caFile;
|
|
# # tlsKeyPath = certs.self.keyFile;
|
|
# # tlsPem = certs.self.certFile;
|
|
# };
|
|
# };
|
|
networking = {
|
|
hostName = name;
|
|
extraHosts = settings.clusterHosts;
|
|
# nameservers = [ masterAddress ];
|
|
# dhcpcd.extraConfig = ''
|
|
# static domain_name_servers=${masterAddress}
|
|
# '';
|
|
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
|
firewall.allowedTCPPorts = [ 80 443 111 ];
|
|
firewall.allowedUDPPorts = [ 111 24007 24008 ];
|
|
};
|
|
};
|
|
|
|
apiserver = ip: name: self:
|
|
{
|
|
deployment.targetHost = ip;
|
|
require = [
|
|
(baseNixos name)
|
|
kubeMaster
|
|
];
|
|
};
|
|
|
|
worker = ip: name: self:
|
|
{
|
|
deployment.targetHost = ip;
|
|
require = [
|
|
(baseNixos name)
|
|
kubeWorker
|
|
];
|
|
};
|
|
|
|
host = ip: name: self:
|
|
{
|
|
deployment.targetHost = ip;
|
|
require = [
|
|
(baseNixos name)
|
|
];
|
|
};
|
|
}
|