Update k0 to new-style nix modules

This commit is contained in:
Jonas Juselius
2021-01-09 13:19:45 +01:00
parent 58dd6e717a
commit ffb2021c85
14 changed files with 112 additions and 41 deletions

1
modules Submodule

Submodule modules added at d471faa617

View File

@@ -1,130 +0,0 @@
{ pkgs, cluster, customize ? {}, extraConfig ? {}, lib, config, ... }:
with lib;
let
cfg = cluster;
mkSANs = host: [
host.name
host.address
"127.0.0.1"
];
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
mkCert = host: {
${host.name} = pki.gencert {
cn = host.name;
ca = cfg.ca;
o = cfg.clusterName;
hosts = [
host.name
"${host.name}.${cfg.domain}"
host.address
"127.0.0.1"
];
};
};
in {
k8s = rec {
apiserver = host: self: {
deployment.targetHost = host.address;
inherit customize;
cluster = mkMerge [
cfg
{
hostName = host.name;
address = host.address;
k8s.master.enable = true;
k8s.node.enable = true;
}
];
services.kubernetes.kubelet.extraSANs = mkSANs host;
imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
};
node = host: self: {
deployment.targetHost = host.address;
inherit customize;
cluster = mkMerge [
cfg
{
hostName = host.name;
address = host.address;
k8s.node.enable = true;
}
];
services.kubernetes.kubelet.extraSANs = mkSANs host;
imports = [ host.hw extraConfig ./modules.nix ../overrides/kubelet.nix ];
};
mkDeployment = top:
let
master = cfg.k8s.master // { hw = "${top}/${master.name}.nix"; };
nodes = cfg.k8s.nodes;
server = { "${master.name}" = apiserver master; };
in
builtins.foldl' (a: x:
a // {
"${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
}) server nodes;
};
fs = rec {
node = host: self: {
deployment.targetHost = host.address;
inherit customize;
cluster = mkMerge [
{ fs.nfs.enable = mkDefault true; }
cfg
{
hostName = host.name;
address = host.address;
cert = mkCert host.name;
}
];
imports = [ host.hw extraConfig ./modules.nix ];
};
mkDeployment = top: nodes:
builtins.foldl' (a: x:
a // {
"${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
}) {} nodes;
} ;
host = rec {
node = host: self: {
deployment.targetHost = host.address;
inherit customize;
cluster = mkMerge [
cfg
{
hostName = host.name;
address = host.address;
cert = mkCert host.name;
}
];
imports = [ host.hw extraConfig ./modules.nix ];
};
mkDeployment = top: nodes:
builtins.foldl' (a: x:
a // {
"${x.name}" = node (x // { hw = "${top}/${x.name}.nix"; });
}) {} nodes;
};
}

View File

@@ -1,67 +0,0 @@
{ pkgs, lib, config, ... }:
with lib;
let
cfg = config.cluster.fs;
cert = cfg.cert;
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
common = {
boot.kernelModules = [
"dm_snapshot"
"dm_mirror"
"dm_thin_pool"
];
networking = {
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
firewall.allowedTCPPorts = [ 111 2049 ];
firewall.allowedUDPPorts = [ 111 2049 24007 24008 ];
};
environment.systemPackages = [ pkgs.lvm2 ];
};
glusterfs = {
services.glusterfs = {
enable = true;
tlsSettings = {
caCert = pki.ca.cert;
tlsKeyPath = cert.key;
tlsPem = cert.cert;
};
};
};
nfs = {
services.nfs.server = {
enable = true;
exports = cfg.nfs.exports;
};
};
in {
options.cluster.fs = {
enable = mkEnableOption "Enable nfs fileserver";
nfs = {
enable = mkEnableOption "Enable nfs fileserver";
exports = mkOption {
type = types.str;
default = "";
};
};
glusterfs.enable = mkEnableOption "Enable glusterfs fileserver";
};
config = mkIf cfg.enable (
mkMerge [
common
(mkIf cfg.nfs.enable nfs)
(mkIf cfg.glusterfs.enable glusterfs)
]
);
imports = [ ./os.nix ];
}

View File

@@ -1,42 +0,0 @@
{
pkgs ? import <nixpkgs> {},
ca ? null,
name ? "ca",
algo ? "rsa",
hosts ? [],
...}:
with pkgs;
let
ca_csr = pkgs.writeText "${name}-csr.json" (builtins.toJSON {
inherit hosts;
CN = "${name}";
key = {
inherit algo;
size = if algo == "ecdsa" then 256 else 2048;
};
names = [
{
CN = "${name}";
O = "NixOS";
OU = "${name}.pki.caSpec";
L = "generated";
}
];
}
);
ca' =
pkgs.runCommand "initca" {
buildInputs = [ pkgs.cfssl ];
} '' cfssl genkey -initca ${ca_csr} | cfssljson -bare ca;
mkdir -p $out; cp *.pem $out '';
initca = if ca != null then ca else ca';
in
# make ca derivation sha depend on initca cfssl output
pkgs.stdenv.mkDerivation {
inherit name;
src = initca;
buildCommand = ''
mkdir -p $out;
cp -r $src/* $out
'';
}

View File

@@ -1,384 +0,0 @@
{ pkgs, lib, config, ...}:
with lib;
let
cfg = config.cluster;
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
masterAddress = cfg.k8s.master.address;
apiserverAddress = "https://${masterAddress}:4443";
cfssl-apitoken =
let
apitoken = pkgs.stdenv.mkDerivation {
name = "apitoken";
buildCommand = ''
head -c ${toString (32 / 2)} /dev/urandom | \
od -An -t x | tr -d ' ' > $out
chmod 400 $out
'';
};
in
# make ca derivation sha depend on initca cfssl output
pkgs.stdenv.mkDerivation {
name = "cfssl-apitoken";
src = apitoken;
buildCommand = ''
cp $src $out
'';
};
cluster-scripts =
let
first = builtins.head cfg.k8s.ingressNodes;
rest = builtins.tail cfg.k8s.ingressNodes;
ingressNodes = builtins.foldl' (a: x:
a + ",${x}") first rest;
ingressReplicaCount =
builtins.toString (builtins.length cfg.k8s.ingressNodes);
show-kubernetes-charts-config = ''
#!/usr/bin/env bash
cat << EOF
# Generated by show-kubernetes-charts-config
# $(date)
# Charts in ${kubernetes-charts}/share/kubernetes-charts
vars=(
initca="${pki.initca}"
apiserver="${cfg.k8s.master.name}"
cluster="${cfg.clusterName}"
ingress_nodes="${ingressNodes}"
ingress_replica_count="${ingressReplicaCount}"
fileserver="${cfg.k8s.fileserver}"
acme_email="${cfg.k8s.charts.acme_email}"
grafana_smtp_user="$(echo -n ${cfg.k8s.charts.grafana_smtp_user} | base64 -w0)"
grafana_smtp_password="$(echo -n ${cfg.k8s.charts.grafana_smtp_password} | base64 -w0)"
)
# -------------------- no modifications below --------------------
EOF
cat << '"'"'EOF'"'"'
apply=''${KUBECTL_CMD:-apply}
substitute_all () {
local x i k v subs
x="$(</dev/stdin)"
subs=("$@")
for i in "''${subs[@]}"; do
k=$(echo "$i" | cut -d= -f1)
v=$(echo "$i" | cut -d= -f2 | sed '"'"'s,/,\\/,g'"'"')
x=$(echo "$x" | sed "s/@$k@/$v/g")
done
echo "$x"
}
substitute_defaults () {
substitute_all "''${vars[@]}"
}
create_namespace () {
kubectl get ns $namespace >/dev/null 2>&1 || kubectl create ns $namespace
}
export_vars () {
local i
for i in "''${vars[@]}"; do eval "$i"; done
}
kubectl_apply () {
local x
x="$(</dev/stdin)"
create_namespace
echo "$x" | \
substitute_defaults | \
kubectl -n $namespace $apply $@ -f -
}
kubectl_apply_files () {
local i charts
charts=("$@")
for i in "''${charts[@]}"; do
cat "$i" | kubectl_apply
done
}
helm_apply () {
local name chart
name="$1"; shift
chart="$1"; shift
create_namespace $namespace
cat values.yaml | \
substitute_defaults | \
helm template -n $namespace -f- $name $chart $@ | \
kubectl -n $namespace $apply -f -
}
[ -z "$namespace" ] && echo "namespace must be set" && exit 1
if [ "$apply" = delete ]; then
echo "WARNING: Really delete $(basename $TOP)? YES/no:"
read ans
[ "$ans" != YES ] && exit 0
fi
export_vars
EOF
'';
copy-kubernetes-charts = ''
#!/usr/bin/env bash
if [ \$# = 0 ]; then
dest=${cfg.clusterName}-charts
else
dest=\$1
fi
[ -e \$dest.bak ] && rm -rf \$dest.bak
[ -e \$dest ] && mv \$dest \$dest.bak
echo "Copying charts into \$dest"
cp -r ${kubernetes-charts}/share/kubernetes-charts \$dest
chown -R \$USER \$dest
chmod -R u+w \$dest
$out/bin/show-kubernetes-charts-config > \$dest/config.sh
'';
in
pkgs.stdenv.mkDerivation {
name = "cluster-scripts";
src = ../scripts;
buildCommand = ''
mkdir -p $out/bin
cp $src/* $out/bin
echo '${show-kubernetes-charts-config}' > $out/bin/show-kubernetes-charts-config
chmod a+x $out/bin/show-kubernetes-charts-config
echo "${copy-kubernetes-charts}" > $out/bin/copy-kubernetes-charts
chmod a+x $out/bin/copy-kubernetes-charts
'';
};
kubernetes-charts = pkgs.stdenv.mkDerivation rec {
name = "kubernetes-charts";
src = ../charts;
buildCommand = ''
mkdir -p $out/share/${name}
cp -r $src/* $out/share/${name}
'';
};
install-apitoken = ''
#!${pkgs.bash}/bin/bash
set -e
if [ -d /var/lib/cfssl ]; then
cp ${cfssl-apitoken} /var/lib/cfssl/apitoken.secret
chown cfssl /var/lib/cfssl/apitoken.secret
chmod 640 /var/lib/cfssl/apitoken.secret
else
mkdir -p /var/lib/kubernetes/secrets
cp ${cfssl-apitoken} /var/lib/kubernetes/secrets/apitoken.secret
chown root /var/lib/kubernetes/secrets/apitoken.secret
chmod 600 /var/lib/kubernetes/secrets/apitoken.secret
fi
'';
kubeMaster = {
services.cfssl.ca = pki.ca.cert;
services.cfssl.caKey = pki.ca.key;
services.kubernetes = {
roles = [ "master" ];
inherit apiserverAddress;
masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
clusterCidr = cfg.k8s.cidr;
pki.genCfsslCACert = false;
pki.genCfsslAPIToken = false;
pki.caCertPathPrefix = "${pki.initca}/ca";
kubelet = {
# clusterDomain = "${cfg.clusterName}.local";
};
apiserver = {
advertiseAddress = masterAddress;
authorizationMode = [ "Node" "RBAC" ];
allowPrivileged = true;
securePort = 4443;
insecurePort = 8080;
extraOpts = "--requestheader-client-ca-file ${pki.ca.cert}";
extraSANs = cfg.k8s.master.extraSANs;
# verbosity = 4;
};
controllerManager = {
bindAddress = masterAddress;
extraOpts = "--authorization-always-allow-paths=/healthz,/metrics";
};
scheduler.address = masterAddress;
addonManager.enable = true;
addons = {
dns = {
enable = true;
# clusterDomain = "${cfg.clusterName}.local";
reconcileMode = "EnsureExists";
};
};
};
services.etcd = {
listenClientUrls = [ "https://${masterAddress}:2379" ];
};
networking.firewall = {
allowedTCPPorts = [ 53 5000 8080 4443 4001 2379 2380 10250 10251 10252 ];
allowedUDPPorts = [ 53 4053 ];
};
environment.systemPackages = [
pkgs.kubernetes-helm
pkgs.kubectl
cluster-scripts
kubernetes-charts
];
systemd.services.kube-certmgr-apitoken-bootstrap = {
description = "Kubernetes certmgr bootstrapper";
wantedBy = [ "cfssl.service" ];
before = [ "cfssl.target" ];
script = install-apitoken;
serviceConfig = {
RestartSec = "10s";
Restart = "on-failure";
};
};
# systemd.services.cfssl-restart = {
# enable = true;
# startAt = "00/6:00";
# description = "Restrart cfssl which regularly locks up";
# script = "systemctl restart cfssl.service";
# };
systemd.services.kube-socat-https-proxy = {
enable = true;
wantedBy = [ "kubernetes.target" ];
after = [ "kubelet.target" ];
description = "Proxy TCP port 443 to ingress NodePort at 30443";
script = "${pkgs.socat}/bin/socat TCP-LISTEN:443,fork,reuseaddr TCP:127.0.0.1:30443";
serviceConfig = {
RestartSec = "10s";
Restart = "on-failure";
};
};
};
kubeNode = {
services.kubernetes = rec {
roles = [ "node" ];
inherit apiserverAddress;
# masterAddress = cfg.k8s.master.name;
masterAddress = "${cfg.k8s.master.name}.${cfg.domain}";
clusterCidr = cfg.k8s.cidr;
# kubelet.clusterDomain = "${cfg.clusterName}.local";
kubelet.hostname = "${cfg.hostName}.${cfg.domain}";
proxy.hostname = "${cfg.hostName}.${cfg.domain}";
};
networking = {
firewall = {
enable = true;
allowedTCPPorts = [ 4194 10250 ];
allowedUDPPorts = [ 53 ];
};
};
virtualisation.docker.extraOptions = "--insecure-registry 10.0.0.0/8";
virtualisation.docker.autoPrune.enable = true;
systemd.services.kube-certmgr-apitoken-bootstrap = {
description = "Kubernetes certmgr bootstrapper";
wantedBy = [ "certmgr.service" ];
before = [ "certmgr.service" ];
script = install-apitoken;
serviceConfig = {
RestartSec = "10s";
Restart = "on-failure";
};
};
};
in {
options.cluster.k8s = {
enable = mkEnableOption "Enable kubernetes";
nodes = mkOption {
type = types.listOf types.attrs;
default = [];
};
fileserver = mkOption {
type = types.str;
default = null;
};
cidr = mkOption {
type = types.str;
default = "10.0.0.0/16";
};
ingressNodes = mkOption {
type = types.listOf types.str;
default = null;
};
master = {
enable = mkEnableOption "Enable kubernetes master node";
address = mkOption {
type = types.str;
default = null;
};
name = mkOption {
type = types.str;
default = null;
};
extraSANs = mkOption {
type = types.listOf types.str;
default = [];
};
hw = mkOption {
type = types.path;
default = null;
};
};
node = {
enable = mkEnableOption "Enable kubernetes";
};
charts = {
acme_email = mkOption {
type = types.str;
default = "";
};
grafana_smtp_user = mkOption {
type = types.str;
default = "";
};
grafana_smtp_password = mkOption {
type = types.str;
default = "";
};
};
};
config = mkIf cfg.k8s.enable (
mkMerge [
(mkIf cfg.k8s.master.enable kubeMaster)
(mkIf cfg.k8s.node.enable kubeNode)
]
);
imports = [ ./os.nix ];
}

View File

@@ -1,7 +0,0 @@
{
imports = [
./os.nix
./fs.nix
./k8s.nix
];
}

View File

@@ -1,118 +0,0 @@
{ pkgs, lib, config, ... }:
with lib;
let
cfg = config.cluster;
pki = import ./pki.nix { inherit pkgs; ca = cfg.initca; };
common = {
users.extraUsers.admin.openssh.authorizedKeys.keys =
cfg.adminAuthorizedKeys;
users.extraUsers.root.openssh.authorizedKeys.keys =
cfg.adminAuthorizedKeys;
networking = {
domain = cfg.domain;
search = cfg.searchDomains;
extraHosts = cfg.extraHosts;
firewall.allowedTCPPortRanges = [ { from = 5000; to = 50000; } ];
firewall.allowedTCPPorts = [ 80 443 111 ];
firewall.allowedUDPPorts = [ 111 24007 24008 ];
} // (
if cfg.externalInterface == null then
{ hostName = cfg.hostName; }
else {
hostName = cfg.hostName;
interfaces."${cfg.externalInterface}" = {
useDHCP = false;
ipv4.addresses = [ {
address = cfg.address;
prefixLength = 24;
} ];
};
defaultGateway = cfg.defaultGateway;
nameservers = cfg.nameservers;
}
);
security.pki.certificateFiles = [ pki.ca.cert ];
boot.kernel.sysctl = {
"kernel.mm.transparent_hugepage.enabled" = "never";
"net.core.somaxconn" = "512";
};
environment.systemPackages = with pkgs; [
nfs-utils
];
};
in
{
options.cluster = {
initca = mkOption {
type = types.path;
};
hostName = mkOption {
type = types.nullOr types.str;
default = null;
};
address = mkOption {
type = types.nullOr types.str;
default = null;
};
externalInterface = mkOption {
type = types.nullOr types.str;
default = null;
};
defaultGateway = mkOption {
type = types.nullOr types.str;
default = null;
};
nameservers = mkOption {
type = types.listOf types.str;
default = [ "8.8.8.8" ];
};
domain = mkOption {
type = types.str;
default = null;
};
searchDomains = mkOption {
type = types.listOf types.str;
default = [ cfg.domain ];
};
cert = mkOption {
type = types.attrs;
default = null;
};
clusterName = mkOption {
type = types.str;
default = null;
};
extraHosts = mkOption {
type = types.str;
};
adminAuthorizedKeys = mkOption {
type = types.listOf types.str;
default = [];
};
};
config = common;
imports = [
../nixos/configuration.nix
];
}

View File

@@ -1,82 +0,0 @@
{ pkgs, ca ? "", algo ? "rsa" }:
let
initca = import ./initca.nix { inherit pkgs ca; };
ca' = {
key = "${initca}/ca-key.pem";
cert = "${initca}/ca.pem";
};
ca-config = pkgs.writeText "ca-config.json" ''
{
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"default": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
'';
gencsr = args:
let
csr = {
CN = "${args.cn}";
key = {
inherit algo;
size = if algo == "ecdsa" then 256 else 2048;
};
names = [
{
CN = "${args.cn}";
O = "${args.o}";
OU = "${args.cn}.${args.o}.pki.caSpec";
L = "generated";
}
];
hosts = args.hosts;
};
in
pkgs.writeText "${args.cn}-csr.json" (builtins.toJSON csr);
in
# Example usage:
#
# gencert { cn = "test"; ca = ca; o = "test; };
#
rec {
inherit initca;
ca = ca';
gencert = attrs:
let
conf = {
cn = attrs.cn;
ca = attrs.ca;
csr = gencsr { cn = attrs.cn; o = attrs.o; hosts = attrs.hosts; };
};
cfssl = conf:
''
cfssl gencert -ca ${ca.cert} -ca-key ${ca.key} \
-config=${ca-config} -profile=default ${conf.csr} | \
cfssljson -bare cert; \
mkdir -p $out; cp *.pem $out
'';
crt =
pkgs.runCommand "${attrs.cn}" {
buildInputs = [ pkgs.cfssl ];
} (cfssl conf);
in
{
key = "${crt}/cert-key.pem";
cert = "${crt}/cert.pem";
};
}