Move modules and overrides to submodule
This commit is contained in:
Jonas Juselius
@@ -1,130 +0,0 @@
|
||||
{ pkgs, cluster, customize ? {}, extraConfig ? {}, lib, config, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = cluster;
|
||||
|
||||
mkSANs = host: [
|
||||
host.name
|
||||
host.address
|
||||
"127.0.0.1"
|
||||
];
|
||||
|
||||
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
|
||||
|
||||
mkCert = host: {
|
||||
${host.name} = pki.gencert {
|
||||
cn = host.name;
|
||||
ca = cfg.ca;
|
||||
o = cfg.clusterName;
|
||||
hosts = [
|
||||
host.name
|
||||
"${host.name}.${cfg.domain}"
|
||||
host.address
|
||||
"127.0.0.1"
|
||||
];
|
||||
};
|
||||
};
|
||||
in {
|
||||
k8s = rec {
|
||||
apiserver = host: self: {
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
inherit customize;
|
||||
|
||||
cluster = mkMerge [
|
||||
cfg
|
||||
{
|
||||
hostName = host.name;
|
||||
address = host.address;
|
||||
k8s.master.enable = true;
|
||||
k8s.node.enable = true;
|
||||
}
|
||||
];
|
||||
|
||||
services.kubernetes.kubelet.extraSANs = mkSANs host;
|
||||
|
||||
imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
|
||||
};
|
||||
|
||||
node = host: self: {
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
inherit customize;
|
||||
|
||||
cluster = mkMerge [
|
||||
cfg
|
||||
{
|
||||
hostName = host.name;
|
||||
address = host.address;
|
||||
k8s.node.enable = true;
|
||||
}
|
||||
];
|
||||
|
||||
services.kubernetes.kubelet.extraSANs = mkSANs host;
|
||||
|
||||
imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
|
||||
};
|
||||
|
||||
mkDeployment = top:
|
||||
let
|
||||
master = cfg.k8s.master // { hw = "${top}/${master.name}.nix"; };
|
||||
nodes = cfg.k8s.nodes;
|
||||
server = { "${master.name}" = apiserver master; };
|
||||
in
|
||||
builtins.foldl' (a: x:
|
||||
a // {
|
||||
"${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
|
||||
}) server nodes;
|
||||
};
|
||||
|
||||
fs = rec {
|
||||
node = host: self: {
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
inherit customize;
|
||||
|
||||
cluster = mkMerge [
|
||||
{ fs.nfs.enable = mkDefault true; }
|
||||
cfg
|
||||
{
|
||||
hostName = host.name;
|
||||
address = host.address;
|
||||
cert = mkCert host.name;
|
||||
}
|
||||
];
|
||||
|
||||
imports = [ host.hw extraConfig ./modules.nix ];
|
||||
};
|
||||
|
||||
mkDeployment = top: nodes:
|
||||
builtins.foldl' (a: x:
|
||||
a // {
|
||||
"${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
|
||||
}) {} nodes;
|
||||
} ;
|
||||
|
||||
host = rec {
|
||||
node = host: self: {
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
inherit customize;
|
||||
|
||||
cluster = mkMerge [
|
||||
cfg
|
||||
{
|
||||
hostName = host.name;
|
||||
address = host.address;
|
||||
cert = mkCert host.name;
|
||||
}
|
||||
];
|
||||
|
||||
imports = [ host.hw extraConfig ./modules.nix ];
|
||||
};
|
||||
|
||||
mkDeployment = top: nodes:
|
||||
builtins.foldl' (a: x:
|
||||
a // {
|
||||
"${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
|
||||
}) {} nodes;
|
||||
};
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
{ pkgs, lib, config, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.cluster.fs;
|
||||
|
||||
cert = cfg.cert;
|
||||
|
||||
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
|
||||
|
||||
common = {
|
||||
boot.kernelModules = [
|
||||
"dm_snapshot"
|
||||
"dm_mirror"
|
||||
"dm_thin_pool"
|
||||
];
|
||||
|
||||
networking = {
|
||||
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
||||
firewall.allowedTCPPorts = [ 111 2049 ];
|
||||
firewall.allowedUDPPorts = [ 111 2049 24007 24008 ];
|
||||
};
|
||||
|
||||
environment.systemPackages = [ pkgs.lvm2 ];
|
||||
};
|
||||
|
||||
glusterfs = {
|
||||
services.glusterfs = {
|
||||
enable = true;
|
||||
tlsSettings = {
|
||||
caCert = pki.ca.cert;
|
||||
tlsKeyPath = cert.key;
|
||||
tlsPem = cert.cert;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
nfs = {
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
exports = cfg.nfs.exports;
|
||||
};
|
||||
};
|
||||
in {
|
||||
options.cluster.fs = {
|
||||
enable = mkEnableOption "Enable nfs fileserver";
|
||||
|
||||
nfs = {
|
||||
enable = mkEnableOption "Enable nfs fileserver";
|
||||
exports = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
};
|
||||
|
||||
glusterfs.enable = mkEnableOption "Enable glusterfs fileserver";
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable (
|
||||
mkMerge [
|
||||
common
|
||||
(mkIf cfg.nfs.enable nfs)
|
||||
(mkIf cfg.glusterfs.enable glusterfs)
|
||||
]
|
||||
);
|
||||
|
||||
imports = [ ./os.nix ];
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
pkgs ? import <nixpkgs> {},
|
||||
ca ? null,
|
||||
name ? "ca",
|
||||
algo ? "rsa",
|
||||
hosts ? [],
|
||||
...}:
|
||||
with pkgs;
|
||||
let
|
||||
ca_csr = pkgs.writeText "${name}-csr.json" (builtins.toJSON {
|
||||
inherit hosts;
|
||||
CN = "${name}";
|
||||
key = {
|
||||
inherit algo;
|
||||
size = if algo == "ecdsa" then 256 else 2048;
|
||||
};
|
||||
names = [
|
||||
{
|
||||
CN = "${name}";
|
||||
O = "NixOS";
|
||||
OU = "${name}.pki.caSpec";
|
||||
L = "generated";
|
||||
}
|
||||
];
|
||||
}
|
||||
);
|
||||
ca' =
|
||||
pkgs.runCommand "initca" {
|
||||
buildInputs = [ pkgs.cfssl ];
|
||||
} '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca;
|
||||
mkdir -p $out; cp *.pem $out '';
|
||||
initca = if ca != null then ca else ca';
|
||||
in
|
||||
# make ca derivation sha depend on initca cfssl output
|
||||
pkgs.stdenv.mkDerivation {
|
||||
inherit name;
|
||||
src = initca;
|
||||
buildCommand = ''
|
||||
mkdir -p $out;
|
||||
cp -r $src/* $out
|
||||
'';
|
||||
}
|
||||
@@ -1,384 +0,0 @@
|
||||
{ pkgs, lib, config, ...}:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.cluster;
|
||||
|
||||
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
|
||||
|
||||
masterAddress = cfg.k8s.master.address;
|
||||
|
||||
apiserverAddress = "https://${masterAddress}:4443";
|
||||
|
||||
cfssl-apitoken =
|
||||
let
|
||||
apitoken = pkgs.stdenv.mkDerivation {
|
||||
name = "apitoken";
|
||||
buildCommand = ''
|
||||
head -c ${toString (32 / 2)} /dev/urandom | \
|
||||
od -An -t x | tr -d ' ' > $out
|
||||
chmod 400 $out
|
||||
'';
|
||||
};
|
||||
in
|
||||
# make ca derivation sha depend on initca cfssl output
|
||||
pkgs.stdenv.mkDerivation {
|
||||
name = "cfssl-apitoken";
|
||||
src = apitoken;
|
||||
buildCommand = ''
|
||||
cp $src $out
|
||||
'';
|
||||
};
|
||||
|
||||
cluster-scripts =
|
||||
let
|
||||
first = builtins.head cfg.k8s.ingressNodes;
|
||||
rest = builtins.tail cfg.k8s.ingressNodes;
|
||||
ingressNodes = builtins.foldl' (a: x:
|
||||
a + ",${x}") first rest;
|
||||
ingressReplicaCount =
|
||||
builtins.toString (builtins.length cfg.k8s.ingressNodes);
|
||||
show-kubernetes-charts-config = ''
|
||||
#!/usr/bin/env bash
|
||||
cat << EOF
|
||||
# Generated by show-kubernetes-charts-config
|
||||
# $(date)
|
||||
# Charts in ${kubernetes-charts}/share/kubernetes-charts
|
||||
|
||||
vars=(
|
||||
initca="${pki.initca}"
|
||||
apiserver="${cfg.k8s.master.name}"
|
||||
cluster="${cfg.clusterName}"
|
||||
ingress_nodes="${ingressNodes}"
|
||||
ingress_replica_count="${ingressReplicaCount}"
|
||||
fileserver="${cfg.k8s.fileserver}"
|
||||
acme_email="${cfg.k8s.charts.acme_email}"
|
||||
grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)"
|
||||
grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)"
|
||||
)
|
||||
|
||||
# -------------------- no modifications below --------------------
|
||||
EOF
|
||||
cat << '"'"'EOF'"'"'
|
||||
apply=''${KUBECTL_CMD:-apply}
|
||||
|
||||
substitute_all () {
|
||||
local x i k v subs
|
||||
x="$(</dev/stdin)"
|
||||
subs=("$@")
|
||||
for i in "''${subs[@]}"; do
|
||||
k=$(echo "$i" | cut -d= -f1)
|
||||
v=$(echo "$i" | cut -d= -f2 | sed '"'"'s,/,\\/,g'"'"')
|
||||
x=$(echo "$x" | sed "s/@$k@/$v/g")
|
||||
done
|
||||
echo "$x"
|
||||
}
|
||||
|
||||
substitute_defaults () {
|
||||
substitute_all "''${vars[@]}"
|
||||
}
|
||||
|
||||
create_namespace () {
|
||||
kubectl get ns $namespace >/dev/null 2>&1 || kubectl create ns $namespace
|
||||
}
|
||||
|
||||
export_vars () {
|
||||
local i
|
||||
for i in "''${vars[@]}"; do eval "$i"; done
|
||||
}
|
||||
|
||||
kubectl_apply () {
|
||||
local x
|
||||
x="$(</dev/stdin)"
|
||||
create_namespace
|
||||
echo "$x" | \
|
||||
substitute_defaults | \
|
||||
kubectl -n $namespace $apply $@ -f -
|
||||
}
|
||||
|
||||
kubectl_apply_files () {
|
||||
local i charts
|
||||
charts=("$@")
|
||||
for i in "''${charts[@]}"; do
|
||||
cat "$i" | kubectl_apply
|
||||
done
|
||||
}
|
||||
|
||||
helm_apply () {
|
||||
local name chart
|
||||
name="$1"; shift
|
||||
chart="$1"; shift
|
||||
create_namespace $namespace
|
||||
cat values.yaml | \
|
||||
substitute_defaults | \
|
||||
helm template -n $namespace -f- $name $chart $@ | \
|
||||
kubectl -n $namespace $apply -f -
|
||||
}
|
||||
|
||||
[ -z "$namespace" ] && echo "namespace must be set" && exit 1
|
||||
|
||||
if [ "$apply" = delete ]; then
|
||||
echo "WARNING: Really delete $(basename $TOP)? YES/no:"
|
||||
read ans
|
||||
[ "$ans" != YES ] && exit 0
|
||||
fi
|
||||
|
||||
export_vars
|
||||
EOF
|
||||
'';
|
||||
copy-kubernetes-charts = ''
|
||||
#!/usr/bin/env bash
|
||||
if [ \$# = 0 ]; then
|
||||
dest=${cfg.clusterName}-charts
|
||||
else
|
||||
dest=\$1
|
||||
fi
|
||||
[ -e \$dest.bak ] && rm -rf \$dest.bak
|
||||
[ -e \$dest ] && mv \$dest \$dest.bak
|
||||
echo "Copying charts into \$dest"
|
||||
cp -r ${kubernetes-charts}/share/kubernetes-charts \$dest
|
||||
chown -R \$USER \$dest
|
||||
chmod -R u+w \$dest
|
||||
$out/bin/show-kubernetes-charts-config > \$dest/config.sh
|
||||
'';
|
||||
in
|
||||
pkgs.stdenv.mkDerivation {
|
||||
name = "cluster-scripts";
|
||||
src = ../scripts;
|
||||
buildCommand = ''
|
||||
mkdir -p $out/bin
|
||||
cp $src/* $out/bin
|
||||
echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config
|
||||
chmod a+x $out/bin/show-kubernetes-charts-config
|
||||
echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts
|
||||
chmod a+x $out/bin/copy-kubernetes-charts
|
||||
'';
|
||||
};
|
||||
|
||||
kubernetes-charts = pkgs.stdenv.mkDerivation rec {
|
||||
name = "kubernetes-charts";
|
||||
src = ../charts;
|
||||
buildCommand = ''
|
||||
mkdir -p $out/share/${name}
|
||||
cp -r $src/* $out/share/${name}
|
||||
'';
|
||||
};
|
||||
|
||||
install-apitoken = ''
|
||||
#!${pkgs.bash}/bin/bash
|
||||
set -e
|
||||
if [ -d /var/lib/cfssl ]; then
|
||||
cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
|
||||
chown cfssl /var/lib/cfssl/apitoken.secret
|
||||
chmod 640 /var/lib/cfssl/apitoken.secret
|
||||
else
|
||||
mkdir -p /var/lib/kubernetes/secrets
|
||||
cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
|
||||
chown root /var/lib/kubernetes/secrets/apitoken.secret
|
||||
chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
|
||||
fi
|
||||
'';
|
||||
|
||||
kubeMaster = {
|
||||
services.cfssl.ca = pki.ca.cert;
|
||||
services.cfssl.caKey = pki.ca.key;
|
||||
services.kubernetes = {
|
||||
roles = [ "master" ];
|
||||
inherit apiserverAddress;
|
||||
masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
|
||||
clusterCidr = cfg.k8s.cidr;
|
||||
pki.genCfsslCACert = false;
|
||||
pki.genCfsslAPIToken = false;
|
||||
pki.caCertPathPrefix = "${pki.initca}/ca";
|
||||
|
||||
kubelet = {
|
||||
# clusterDomain = "${cfg.clusterName}.local";
|
||||
};
|
||||
|
||||
apiserver = {
|
||||
advertiseAddress = masterAddress;
|
||||
authorizationMode = [ "Node" "RBAC" ];
|
||||
allowPrivileged = true;
|
||||
securePort = 4443;
|
||||
insecurePort = 8080;
|
||||
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
|
||||
extraSANs = cfg.k8s.master.extraSANs;
|
||||
# verbosity = 4;
|
||||
};
|
||||
|
||||
controllerManager = {
|
||||
bindAddress = masterAddress;
|
||||
extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
|
||||
};
|
||||
|
||||
scheduler.address = masterAddress;
|
||||
|
||||
addonManager.enable = true;
|
||||
addons = {
|
||||
dns = {
|
||||
enable = true;
|
||||
# clusterDomain = "${cfg.clusterName}.local";
|
||||
reconcileMode = "EnsureExists";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.etcd = {
|
||||
listenClientUrls = [ "https://${masterAddress}:2379" ];
|
||||
};
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
|
||||
allowedUDPPorts = [ 53 4053 ];
|
||||
};
|
||||
|
||||
environment.systemPackages = [
|
||||
pkgs.kubernetes-helm
|
||||
pkgs.kubectl
|
||||
cluster-scripts
|
||||
kubernetes-charts
|
||||
];
|
||||
|
||||
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
||||
description = "Kubernetes certmgr bootstrapper";
|
||||
wantedBy = [ "cfssl.service" ];
|
||||
before = [ "cfssl.target" ];
|
||||
script = install-apitoken;
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
|
||||
# systemd.services.cfssl-restart = {
|
||||
# enable = true;
|
||||
# startAt = "00/6:00";
|
||||
# description = "Restrart cfssl which regularly locks up";
|
||||
# script = "systemctl restart cfssl.service";
|
||||
# };
|
||||
|
||||
systemd.services.kube-socat-https-proxy = {
|
||||
enable = true;
|
||||
wantedBy = [ "kubernetes.target" ];
|
||||
after = [ "kubelet.target" ];
|
||||
description = "Proxy TCP port 443 to ingress NodePort at 30443";
|
||||
script = "${pkgs.socat}/bin/socat TCP-LISTEN:443,fork,reuseaddr TCP:127.0.0.1:30443";
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
kubeNode = {
|
||||
services.kubernetes = rec {
|
||||
roles = [ "node" ];
|
||||
inherit apiserverAddress;
|
||||
# masterAddress = cfg.k8s.master.name;
|
||||
masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
|
||||
clusterCidr = cfg.k8s.cidr;
|
||||
# kubelet.clusterDomain = "${cfg.clusterName}.local";
|
||||
kubelet.hostname = "${cfg.hostName}.${cfg.domain}";
|
||||
proxy.hostname = "${cfg.hostName}.${cfg.domain}";
|
||||
};
|
||||
|
||||
networking = {
|
||||
firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [ 4194 10250 ];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
};
|
||||
};
|
||||
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
|
||||
virtualisation.docker.autoPrune.enable = true;
|
||||
systemd.services.kube-certmgr-apitoken-bootstrap = {
|
||||
description = "Kubernetes certmgr bootstrapper";
|
||||
wantedBy = [ "certmgr.service" ];
|
||||
before = [ "certmgr.service" ];
|
||||
script = install-apitoken;
|
||||
serviceConfig = {
|
||||
RestartSec = "10s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
options.cluster.k8s = {
|
||||
enable = mkEnableOption "Enable kubernetes";
|
||||
nodes = mkOption {
|
||||
type = types.listOf types.attrs;
|
||||
default = [];
|
||||
};
|
||||
|
||||
fileserver = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
cidr = mkOption {
|
||||
type = types.str;
|
||||
default = "10.0.0.0/16";
|
||||
};
|
||||
|
||||
ingressNodes = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
master = {
|
||||
enable = mkEnableOption "Enable kubernetes master node";
|
||||
|
||||
address = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
extraSANs = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
hw = mkOption {
|
||||
type = types.path;
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
|
||||
node = {
|
||||
enable = mkEnableOption "Enable kubernetes";
|
||||
};
|
||||
|
||||
charts = {
|
||||
acme_email = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
|
||||
grafana_smtp_user = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
|
||||
grafana_smtp_password = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.k8s.enable (
|
||||
mkMerge [
|
||||
(mkIf cfg.k8s.master.enable kubeMaster)
|
||||
(mkIf cfg.k8s.node.enable kubeNode)
|
||||
]
|
||||
);
|
||||
|
||||
imports = [ ./os.nix ];
|
||||
|
||||
}
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
imports = [
|
||||
./os.nix
|
||||
./fs.nix
|
||||
./k8s.nix
|
||||
];
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
{ pkgs, lib, config, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.cluster;
|
||||
|
||||
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
|
||||
|
||||
common = {
|
||||
users.extraUsers.admin.openssh.authorizedKeys.keys =
|
||||
cfg.adminAuthorizedKeys;
|
||||
|
||||
users.extraUsers.root.openssh.authorizedKeys.keys =
|
||||
cfg.adminAuthorizedKeys;
|
||||
|
||||
networking = {
|
||||
domain = cfg.domain;
|
||||
search = cfg.searchDomains;
|
||||
extraHosts = cfg.extraHosts;
|
||||
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
|
||||
firewall.allowedTCPPorts = [ 80 443 111 ];
|
||||
firewall.allowedUDPPorts = [ 111 24007 24008 ];
|
||||
} // (
|
||||
if cfg.externalInterface == null then
|
||||
{ hostName = cfg.hostName; }
|
||||
else {
|
||||
hostName = cfg.hostName;
|
||||
interfaces."${cfg.externalInterface}" = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = cfg.address;
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
defaultGateway = cfg.defaultGateway;
|
||||
nameservers = cfg.nameservers;
|
||||
}
|
||||
);
|
||||
|
||||
security.pki.certificateFiles = [ pki.ca.cert ];
|
||||
boot.kernel.sysctl = {
|
||||
"kernel.mm.transparent_hugepage.enabled" = "never";
|
||||
"net.core.somaxconn" = "512";
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
nfs-utils
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
options.cluster = {
|
||||
initca = mkOption {
|
||||
type = types.path;
|
||||
};
|
||||
|
||||
hostName = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
address = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
externalInterface = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
|
||||
defaultGateway = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
nameservers = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ "8.8.8.8" ];
|
||||
};
|
||||
|
||||
domain = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
searchDomains = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ cfg.domain ];
|
||||
};
|
||||
|
||||
cert = mkOption {
|
||||
type = types.attrs;
|
||||
default = null;
|
||||
};
|
||||
|
||||
clusterName = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
extraHosts = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
|
||||
adminAuthorizedKeys = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
};
|
||||
|
||||
config = common;
|
||||
|
||||
imports = [
|
||||
../nixos/configuration.nix
|
||||
];
|
||||
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
{ pkgs, ca ? "", algo ? "rsa" }:
|
||||
let
|
||||
initca = import ./initca.nix { inherit pkgs ca; };
|
||||
|
||||
ca' = {
|
||||
key = "${initca}/ca-key.pem";
|
||||
cert = "${initca}/ca.pem";
|
||||
};
|
||||
|
||||
ca-config = pkgs.writeText "ca-config.json" ''
|
||||
{
|
||||
"signing": {
|
||||
"default": {
|
||||
"expiry": "8760h"
|
||||
},
|
||||
"profiles": {
|
||||
"default": {
|
||||
"usages": [
|
||||
"signing",
|
||||
"key encipherment",
|
||||
"server auth",
|
||||
"client auth"
|
||||
],
|
||||
"expiry": "8760h"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'';
|
||||
|
||||
gencsr = args:
|
||||
let
|
||||
csr = {
|
||||
CN = "${args.cn}";
|
||||
key = {
|
||||
inherit algo;
|
||||
size = if algo == "ecdsa" then 256 else 2048;
|
||||
};
|
||||
names = [
|
||||
{
|
||||
CN = "${args.cn}";
|
||||
O = "${args.o}";
|
||||
OU = "${args.cn}.${args.o}.pki.caSpec";
|
||||
L = "generated";
|
||||
}
|
||||
];
|
||||
hosts = args.hosts;
|
||||
};
|
||||
in
|
||||
pkgs.writeText "${args.cn}-csr.json" (builtins.toJSON csr);
|
||||
in
|
||||
# Example usage:
|
||||
#
|
||||
# gencert { cn = "test"; ca = ca; o = "test; };
|
||||
#
|
||||
rec {
|
||||
inherit initca;
|
||||
ca = ca';
|
||||
gencert = attrs:
|
||||
let
|
||||
conf = {
|
||||
cn = attrs.cn;
|
||||
ca = attrs.ca;
|
||||
csr = gencsr { cn = attrs.cn; o = attrs.o; hosts = attrs.hosts; };
|
||||
};
|
||||
cfssl = conf:
|
||||
''
|
||||
cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
|
||||
-config=${ca-config} -profile=default ${conf.csr} | \
|
||||
cfssljson -bare cert; \
|
||||
mkdir -p $out; cp *.pem $out
|
||||
'';
|
||||
crt =
|
||||
pkgs.runCommand "${attrs.cn}" {
|
||||
buildInputs = [ pkgs.cfssl ];
|
||||
} (cfssl conf);
|
||||
in
|
||||
{
|
||||
key = "${crt}/cert-key.pem";
|
||||
cert = "${crt}/cert.pem";
|
||||
};
|
||||
}
|
||||
@@ -1,359 +0,0 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
top = config.services.kubernetes;
|
||||
cfg = top.kubelet;
|
||||
|
||||
cniConfig =
|
||||
if cfg.cni.config != [] && cfg.cni.configDir != null then
|
||||
throw "Verbatim CNI-config and CNI configDir cannot both be set."
|
||||
else if cfg.cni.configDir != null then
|
||||
cfg.cni.configDir
|
||||
else
|
||||
(pkgs.buildEnv {
|
||||
name = "kubernetes-cni-config";
|
||||
paths = imap (i: entry:
|
||||
pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
|
||||
) cfg.cni.config;
|
||||
});
|
||||
|
||||
infraContainer = pkgs.dockerTools.buildImage {
|
||||
name = "pause";
|
||||
tag = "latest";
|
||||
contents = top.package.pause;
|
||||
config.Cmd = "/bin/pause";
|
||||
};
|
||||
|
||||
kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
|
||||
|
||||
manifestPath = "kubernetes/manifests";
|
||||
|
||||
taintOptions = with lib.types; { name, ... }: {
|
||||
options = {
|
||||
key = mkOption {
|
||||
description = "Key of taint.";
|
||||
default = name;
|
||||
type = str;
|
||||
};
|
||||
value = mkOption {
|
||||
description = "Value of taint.";
|
||||
type = str;
|
||||
};
|
||||
effect = mkOption {
|
||||
description = "Effect of taint.";
|
||||
example = "NoSchedule";
|
||||
type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints);
|
||||
in
|
||||
{
|
||||
disabledModules = [ "services/cluster/kubernetes/kubelet.nix" ];
|
||||
|
||||
imports = [
|
||||
(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
|
||||
(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
|
||||
(mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
|
||||
];
|
||||
|
||||
###### interface
|
||||
options.services.kubernetes.kubelet = with lib.types; {
|
||||
|
||||
address = mkOption {
|
||||
description = "Kubernetes kubelet info server listening address.";
|
||||
default = "0.0.0.0";
|
||||
type = str;
|
||||
};
|
||||
|
||||
clusterDns = mkOption {
|
||||
description = "Use alternative DNS.";
|
||||
default = "10.1.0.1";
|
||||
type = str;
|
||||
};
|
||||
|
||||
clusterDomain = mkOption {
|
||||
description = "Use alternative domain.";
|
||||
default = config.services.kubernetes.addons.dns.clusterDomain;
|
||||
type = str;
|
||||
};
|
||||
|
||||
extraSANs = mkOption {
|
||||
description = "Extra x509 Subject Alternative Names to be added to the kubelet tls cert.";
|
||||
default = [];
|
||||
type = listOf str;
|
||||
};
|
||||
|
||||
clientCaFile = mkOption {
|
||||
description = "Kubernetes apiserver CA file for client authentication.";
|
||||
default = top.caFile;
|
||||
type = nullOr path;
|
||||
};
|
||||
|
||||
cni = {
|
||||
packages = mkOption {
|
||||
description = "List of network plugin packages to install.";
|
||||
type = listOf package;
|
||||
default = [];
|
||||
};
|
||||
|
||||
config = mkOption {
|
||||
description = "Kubernetes CNI configuration.";
|
||||
type = listOf attrs;
|
||||
default = [];
|
||||
example = literalExample ''
|
||||
[{
|
||||
"cniVersion": "0.3.1",
|
||||
"name": "mynet",
|
||||
"type": "bridge",
|
||||
"bridge": "cni0",
|
||||
"isGateway": true,
|
||||
"ipMasq": true,
|
||||
"ipam": {
|
||||
"type": "host-local",
|
||||
"subnet": "10.22.0.0/16",
|
||||
"routes": [
|
||||
{ "dst": "0.0.0.0/0" }
|
||||
]
|
||||
}
|
||||
} {
|
||||
"cniVersion": "0.3.1",
|
||||
"type": "loopback"
|
||||
}]
|
||||
'';
|
||||
};
|
||||
|
||||
configDir = mkOption {
|
||||
description = "Path to Kubernetes CNI configuration directory.";
|
||||
type = nullOr path;
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
|
||||
enable = mkEnableOption "Kubernetes kubelet.";
|
||||
|
||||
extraOpts = mkOption {
|
||||
description = "Kubernetes kubelet extra command line options.";
|
||||
default = "";
|
||||
type = str;
|
||||
};
|
||||
|
||||
featureGates = mkOption {
|
||||
description = "List set of feature gates";
|
||||
default = top.featureGates;
|
||||
type = listOf str;
|
||||
};
|
||||
|
||||
healthz = {
|
||||
bind = mkOption {
|
||||
description = "Kubernetes kubelet healthz listening address.";
|
||||
default = "127.0.0.1";
|
||||
type = str;
|
||||
};
|
||||
|
||||
port = mkOption {
|
||||
description = "Kubernetes kubelet healthz port.";
|
||||
default = 10248;
|
||||
type = int;
|
||||
};
|
||||
};
|
||||
|
||||
hostname = mkOption {
|
||||
description = "Kubernetes kubelet hostname override.";
|
||||
default = config.networking.hostName;
|
||||
type = str;
|
||||
};
|
||||
|
||||
kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
|
||||
|
||||
manifests = mkOption {
|
||||
description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
|
||||
type = attrsOf attrs;
|
||||
default = {};
|
||||
};
|
||||
|
||||
networkPlugin = mkOption {
|
||||
description = "Network plugin to use by Kubernetes.";
|
||||
type = nullOr (enum ["cni" "kubenet"]);
|
||||
default = "kubenet";
|
||||
};
|
||||
|
||||
nodeIp = mkOption {
|
||||
description = "IP address of the node. If set, kubelet will use this IP address for the node.";
|
||||
default = null;
|
||||
type = nullOr str;
|
||||
};
|
||||
|
||||
registerNode = mkOption {
|
||||
description = "Whether to auto register kubelet with API server.";
|
||||
default = true;
|
||||
type = bool;
|
||||
};
|
||||
|
||||
port = mkOption {
|
||||
description = "Kubernetes kubelet info server listening port.";
|
||||
default = 10250;
|
||||
type = int;
|
||||
};
|
||||
|
||||
seedDockerImages = mkOption {
|
||||
description = "List of docker images to preload on system";
|
||||
default = [];
|
||||
type = listOf package;
|
||||
};
|
||||
|
||||
taints = mkOption {
|
||||
description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
|
||||
default = {};
|
||||
type = attrsOf (submodule [ taintOptions ]);
|
||||
};
|
||||
|
||||
tlsCertFile = mkOption {
|
||||
description = "File containing x509 Certificate for HTTPS.";
|
||||
default = null;
|
||||
type = nullOr path;
|
||||
};
|
||||
|
||||
tlsKeyFile = mkOption {
|
||||
description = "File containing x509 private key matching tlsCertFile.";
|
||||
default = null;
|
||||
type = nullOr path;
|
||||
};
|
||||
|
||||
unschedulable = mkOption {
|
||||
description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
|
||||
default = false;
|
||||
type = bool;
|
||||
};
|
||||
|
||||
verbosity = mkOption {
|
||||
description = ''
|
||||
Optional glog verbosity level for logging statements. See
|
||||
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
|
||||
'';
|
||||
default = null;
|
||||
type = nullOr int;
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
###### implementation
|
||||
config = mkMerge [
|
||||
(mkIf cfg.enable {
|
||||
services.kubernetes.kubelet.seedDockerImages = [infraContainer];
|
||||
|
||||
systemd.services.kubelet = {
|
||||
description = "Kubernetes Kubelet Service";
|
||||
wantedBy = [ "kubernetes.target" ];
|
||||
after = [ "network.target" "docker.service" "kube-apiserver.service" ];
|
||||
path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
|
||||
preStart = ''
|
||||
${concatMapStrings (img: ''
|
||||
echo "Seeding docker image: ${img}"
|
||||
docker load <${img}
|
||||
'') cfg.seedDockerImages}
|
||||
|
||||
rm /opt/cni/bin/* || true
|
||||
${concatMapStrings (package: ''
|
||||
echo "Linking cni package: ${package}"
|
||||
ln -fs ${package}/bin/* /opt/cni/bin
|
||||
'') cfg.cni.packages}
|
||||
'';
|
||||
serviceConfig = {
|
||||
Slice = "kubernetes.slice";
|
||||
CPUAccounting = true;
|
||||
MemoryAccounting = true;
|
||||
Restart = "on-failure";
|
||||
RestartSec = "1000ms";
|
||||
ExecStart = ''${top.package}/bin/kubelet \
|
||||
--address=${cfg.address} \
|
||||
--authentication-token-webhook \
|
||||
--authentication-token-webhook-cache-ttl="10s" \
|
||||
--authorization-mode=Webhook \
|
||||
${optionalString (cfg.clientCaFile != null)
|
||||
"--client-ca-file=${cfg.clientCaFile}"} \
|
||||
${optionalString (cfg.clusterDns != "")
|
||||
"--cluster-dns=${cfg.clusterDns}"} \
|
||||
${optionalString (cfg.clusterDomain != "")
|
||||
"--cluster-domain=${cfg.clusterDomain}"} \
|
||||
--cni-conf-dir=${cniConfig} \
|
||||
${optionalString (cfg.featureGates != [])
|
||||
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
|
||||
--hairpin-mode=hairpin-veth \
|
||||
--healthz-bind-address=${cfg.healthz.bind} \
|
||||
--healthz-port=${toString cfg.healthz.port} \
|
||||
--hostname-override=${cfg.hostname} \
|
||||
--kubeconfig=${kubeconfig} \
|
||||
${optionalString (cfg.networkPlugin != null)
|
||||
"--network-plugin=${cfg.networkPlugin}"} \
|
||||
${optionalString (cfg.nodeIp != null)
|
||||
"--node-ip=${cfg.nodeIp}"} \
|
||||
--pod-infra-container-image=pause \
|
||||
${optionalString (cfg.manifests != {})
|
||||
"--pod-manifest-path=/etc/${manifestPath}"} \
|
||||
--port=${toString cfg.port} \
|
||||
--register-node=${boolToString cfg.registerNode} \
|
||||
${optionalString (taints != "")
|
||||
"--register-with-taints=${taints}"} \
|
||||
--root-dir=${top.dataDir} \
|
||||
${optionalString (cfg.tlsCertFile != null)
|
||||
"--tls-cert-file=${cfg.tlsCertFile}"} \
|
||||
${optionalString (cfg.tlsKeyFile != null)
|
||||
"--tls-private-key-file=${cfg.tlsKeyFile}"} \
|
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
|
||||
${cfg.extraOpts}
|
||||
'';
|
||||
WorkingDirectory = top.dataDir;
|
||||
};
|
||||
};
|
||||
|
||||
# Allways include cni plugins
|
||||
services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
|
||||
|
||||
boot.kernelModules = ["br_netfilter"];
|
||||
|
||||
services.kubernetes.kubelet.hostname = with config.networking;
|
||||
mkDefault (hostName + optionalString (domain != null) ".${domain}");
|
||||
|
||||
services.kubernetes.pki.certs = with top.lib; {
|
||||
kubelet = mkCert {
|
||||
name = "kubelet";
|
||||
CN = top.kubelet.hostname;
|
||||
hosts = top.kubelet.extraSANs;
|
||||
action = "systemctl restart kubelet.service";
|
||||
|
||||
};
|
||||
kubeletClient = mkCert {
|
||||
name = "kubelet-client";
|
||||
CN = "system:node:${top.kubelet.hostname}";
|
||||
fields = {
|
||||
O = "system:nodes";
|
||||
};
|
||||
action = "systemctl restart kubelet.service";
|
||||
};
|
||||
};
|
||||
|
||||
services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
|
||||
})
|
||||
|
||||
(mkIf (cfg.enable && cfg.manifests != {}) {
|
||||
environment.etc = mapAttrs' (name: manifest:
|
||||
nameValuePair "${manifestPath}/${name}.json" {
|
||||
text = builtins.toJSON manifest;
|
||||
mode = "0755";
|
||||
}
|
||||
) cfg.manifests;
|
||||
})
|
||||
|
||||
(mkIf (cfg.unschedulable && cfg.enable) {
|
||||
services.kubernetes.kubelet.taints.unschedulable = {
|
||||
value = "true";
|
||||
effect = "NoSchedule";
|
||||
};
|
||||
})
|
||||
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user