oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: enable console on ttyS0 for IPMI SOL

Browse Source
This commit is contained in:
Administrator
2025-11-11 18:28:36 +01:00
parent 4a33496824
commit 059fa13f3b
1 changed files with 379 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

738
ekman/manage/default.nix
View File

@@ -1,388 +1,408 @@
{ pkgs, ...}:
{ pkgs, ... }:
let
computeNodes =
import ../c0/nodes.nix ++
import ../c1/nodes.nix ++
[
rec {
idx = 100;
name = "ekman";
address = "10.255.241.${toString idx}";
ipoib = "10.255.243.${toString idx}";
pubkey = ../login/ekman.pub;
}
rec {
idx = 90;
name = "fs-work";
address = "10.255.241.${toString idx}";
ipoib = "10.255.243.${toString idx}";
pubkey = ../fs-work/fs-work.pub;
}
rec {
idx = 81;
name = "fs-backup";
address = "10.255.241.${toString idx}";
ipoib = "10.255.243.${toString idx}";
pubkey = ../fs-backup/fs-backup.pub;
}
];
import ../c0/nodes.nix
++ import ../c1/nodes.nix
++ [
rec {
idx = 100;
name = "ekman";
address = "10.255.241.${toString idx}";
ipoib = "10.255.243.${toString idx}";
pubkey = ../login/ekman.pub;
}
rec {
idx = 90;
name = "fs-work";
address = "10.255.241.${toString idx}";
ipoib = "10.255.243.${toString idx}";
pubkey = ../fs-work/fs-work.pub;
}
rec {
idx = 81;
name = "fs-backup";
address = "10.255.241.${toString idx}";
ipoib = "10.255.243.${toString idx}";
pubkey = ../fs-backup/fs-backup.pub;
}
];
etcdCluster = import ../etcdCluster.nix;
name = "ekman-manage";
address = "10.255.241.99";
ipoib = "10.255.243.99";
in {
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
in
{
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
};
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
environment.systemPackages = with pkgs; [
rdma-core
hwloc
headscale
];
cluster = {
k8sNode = true;
compute = false;
slurm = true;
mounts = {
rdma.enable = true;
automount.enable = true;
users = false;
opt = true;
work = false;
data = true;
ceph = true;
backup = false;
};
};
features = {
desktop.enable = false;
cachix.enable = false;
host = {
inherit address;
inherit name;
};
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
myvnc.enable = false;
environment.systemPackages = with pkgs; [
rdma-core
hwloc
headscale
];
cluster = {
k8sNode = true;
compute = false;
slurm = true;
mounts = {
rdma.enable = true;
automount.enable = true;
users = false;
opt = true;
work = false;
data = true;
ceph = true;
backup = false;
};
};
features = {
desktop.enable = false;
cachix.enable = false;
host = {
inherit address;
inherit name;
};
myvnc.enable = false;
os = {
externalInterface = "eno1";
networkmanager.enable = false;
nfs.enable = false;
nfs.exports = ''
/exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
hpc = {
slurm.server = true;
slurm.slurmrestd = true;
slurm.dbdServer = false;
manageNode = true;
};
k8s = {
master.enable = true;
node.enable = true;
nodes = computeNodes;
inherit etcdCluster;
};
monitoring = {
server = {
enable = false;
scrapeHosts = [
"ekman-manage"
"ekman"
"fs-work"
"fs-backup"
] ++ (builtins.map (x: x.name) computeNodes);
defaultAlertReceiver = {
email_configs = [
{ to = "jonas.juselius@oceanbox.io"; }
];
};
pageAlertReceiver = {
webhook_configs = [
{
url = "https://prometheus-msteams.k2.itpartner.no/ekman";
http_config = {
tls_config = { insecure_skip_verify = true; };
};
}
];
};
};
webUI.enable = false;
webUI.acmeEmail = "acme@oceanbox.io";
webUI.allow = [
"10.1.2.0/24"
"172.19.254.0/24"
"172.19.255.0/24"
];
infiniband-exporter = {
enable = true;
nameMap = ''
0xe8ebd3030024a2c6 "ekman"
0x0c42a10300ddc4bc "ekman-manage"
0xe8ebd3030024a2ae "fs-work"
0x1c34da0300787798 "fs-backup"
0xe8ebd3030024981e "c0-1"
0xe8ebd3030024a21a "c0-2"
0xe8ebd30300249a3a "c0-3"
0xe8ebd30300248b9e "c0-4"
0xe8ebd30300248b86 "c0-5"
0xe8ebd3030024998a "c0-6"
0xe8ebd30300248b8e "c0-7"
0xe8ebd3030024999e "c0-8"
0xe8ebd30300248fca "c0-9"
0xe8ebd3030024a216 "c0-10"
0xe8ebd30300248b96 "c0-11"
0xe8ebd30300248b9a "c0-12"
0xe8ebd303002495d2 "c0-13"
0xe8ebd303002495e2 "c0-14"
0xe8ebd30300248f42 "c0-15"
0xe8ebd303002495e6 "c0-16"
0x0c42a10300dbe7f4 "c1-1"
0x0c42a10300dbe7d8 "c1-2"
0x0c42a10300dbe800 "c1-3"
0x0c42a10300dbec80 "c1-4"
0x0c42a10300dbea50 "c1-5"
0x0c42a10300dbeb2c "c1-6"
0x0c42a10300dbe7fc "c1-7"
0x0c42a10300dbe5a0 "c1-8"
'';
};
slurm-exporter = {
enable = true;
port = 6080;
};
};
};
programs.singularity.enable = true;
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
services.flannel.iface = "eno2";
networking = {
useNetworkd = true;
hostName = name;
firewall = {
checkReversePath = "loose";
trustedInterfaces = [ "tailscale0" ];
allowedUDPPorts = [ 41641 53 ];
allowedTCPPorts = [ 6443 4725 53 ];
extraCommands = ''
# needed for nodeport access on k1 and k2
# iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
# iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -j SNAT --to-source 10.255.242.3
'';
};
};
systemd.network = {
networks = {
eno1 = {
DHCP = "no";
matchConfig.Name = "eno1";
address = [ "10.255.242.3/24" ];
routes = [ { Gateway = "10.255.242.1"; } ];
};
eno2 = {
DHCP = "no";
matchConfig.Name = "eno2";
address = [ "${address}/24" ];
routes = [
{
Destination = "172.16.239.0/24";
Gateway = "10.255.241.210";
}
];
};
ens2f1np1 = {
DHCP = "no";
matchConfig.Name = "ens2f1np1";
address = [ "10.255.244.99/24" ];
};
ibs2f0 = {
DHCP = "no";
matchConfig.Name = "ibs2f0";
address = [ "${ipoib}/24" ];
};
};
};
services.resolved = {
# DNS=[::1]:53
extraConfig = ''
DNSStubListener=no
os = {
externalInterface = "eno1";
networkmanager.enable = false;
nfs.enable = false;
nfs.exports = ''
/exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
fileSystems = {
"/exports/public" = {
device = "/srv/public";
options = [ "bind" ];
};
hpc = {
slurm.server = true;
slurm.slurmrestd = true;
slurm.dbdServer = false;
manageNode = true;
};
nix.extraOptions = ''
# secret-key-files = /etc/nix/ekman.key
'';
k8s = {
master.enable = true;
node.enable = true;
nodes = computeNodes;
inherit etcdCluster;
};
services.prometheus.alertmanager.configuration.global = {
smtp_smart_host = "oceanbox-io.mail.protection.outlook.com";
smtp_tls_config = {
ca_file = "/etc/ssl/certs/ca-bundle.crt";
cert_file = "/var/lib/secrets/ekman-lb.oceanbox.io.pem";
key_file = "/var/lib/secrets/ekman-lb.oceanbox.io-key.pem";
monitoring = {
server = {
enable = false;
scrapeHosts = [
"ekman-manage"
"ekman"
"fs-work"
"fs-backup"
]
++ (builtins.map (x: x.name) computeNodes);
defaultAlertReceiver = {
email_configs = [
{ to = "jonas.juselius@oceanbox.io"; }
];
};
smtp_hello = "ekman.oceanbox.io";
smtp_from = "noreply@oceanbox.io";
};
security.pam = {
services.sshd.googleAuthenticator.enable = true;
loginLimits = [
{
domain = "@users";
item = "rss";
type = "hard";
value = 16000000;
}
{
domain = "@users";
item = "cpu";
type = "hard";
value = 180;
}
];
};
system.activationScripts = {
home-permissions.text = ''
chmod 755 /home/olean
chmod 755 /home/frankgaa
chmod 755 /home/jonas
chmod 755 /home/stig
chmod 755 /home/bast
chmod 755 /home/mrtz
chmod 755 /home/avle
chmod 755 /home/simenlk
chmod 755 /home/ole
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it
services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
PubkeyAuthOptions verify-required
'';
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
virtualisation.docker.enable = pkgs.lib.mkForce true;
security.acme = {
acceptTerms = true;
defaults.email = "acme@oceanbox.io";
defaults.dnsPropagationCheck = false;
certs."ekman-lb.oceanbox.io" = {
dnsProvider = "namecheap";
environmentFile = "/var/lib/secrets/namecheap.key";
dnsPropagationCheck = false;
postRun = ''
cp fullchain.pem /var/lib/secrets/ekman-lb.oceanbox.io.pem
cp key.pem /var/lib/secrets/ekman-lb.oceanbox.io-key.pem
chmod 644 /var/lib/secrets/ekman-lb.oceanbox.io.pem
chmod 640 /var/lib/secrets/ekman-lb.oceanbox.io-key.pem
'';
};
};
programs.msmtp = {
enable = true;
accounts = {
default = {
auth = false;
host = "oceanbox-io.mail.protection.outlook.com";
port = 25;
from = "noreply@ekman.oceanbox.io";
tls_starttls = true;
tls = true;
tls_trust_file = "/etc/ssl/certs/ca-bundle.crt";
tls_key_file = "/var/lib/secrets/ekman-lb.oceanbox.io-key.pem";
tls_cert_file = "/var/lib/secrets/ekman-lb.oceanbox.io.pem";
pageAlertReceiver = {
webhook_configs = [
{
url = "https://prometheus-msteams.k2.itpartner.no/ekman";
http_config = {
tls_config = {
insecure_skip_verify = true;
};
};
}
];
};
};
defaults = {
aliases = "/etc/aliases";
};
};
environment.etc = {
"aliases" = {
text = ''
root: jonas.juselius@oceanbox.io, moritz.jorg@oceanbox.io
'';
mode = "0644";
};
};
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "both"; # for exit-node usage
extraUpFlags = [
"--login-server=https://headscale.svc.oceanbox.io"
"--accept-dns=true"
"--accept-routes=false"
"--advertise-tags=tag:hpc"
webUI.enable = false;
webUI.acmeEmail = "acme@oceanbox.io";
webUI.allow = [
"10.1.2.0/24"
"172.19.254.0/24"
"172.19.255.0/24"
];
infiniband-exporter = {
enable = true;
nameMap = ''
0xe8ebd3030024a2c6 "ekman"
0x0c42a10300ddc4bc "ekman-manage"
0xe8ebd3030024a2ae "fs-work"
0x1c34da0300787798 "fs-backup"
0xe8ebd3030024981e "c0-1"
0xe8ebd3030024a21a "c0-2"
0xe8ebd30300249a3a "c0-3"
0xe8ebd30300248b9e "c0-4"
0xe8ebd30300248b86 "c0-5"
0xe8ebd3030024998a "c0-6"
0xe8ebd30300248b8e "c0-7"
0xe8ebd3030024999e "c0-8"
0xe8ebd30300248fca "c0-9"
0xe8ebd3030024a216 "c0-10"
0xe8ebd30300248b96 "c0-11"
0xe8ebd30300248b9a "c0-12"
0xe8ebd303002495d2 "c0-13"
0xe8ebd303002495e2 "c0-14"
0xe8ebd30300248f42 "c0-15"
0xe8ebd303002495e6 "c0-16"
0x0c42a10300dbe7f4 "c1-1"
0x0c42a10300dbe7d8 "c1-2"
0x0c42a10300dbe800 "c1-3"
0x0c42a10300dbec80 "c1-4"
0x0c42a10300dbea50 "c1-5"
0x0c42a10300dbeb2c "c1-6"
0x0c42a10300dbe7fc "c1-7"
0x0c42a10300dbe5a0 "c1-8"
'';
};
slurm-exporter = {
enable = true;
port = 6080;
};
};
services.networkd-dispatcher = {
enable = true;
rules = {
"tailscale-router" = {
onState = [ "routable" ];
script = ''
};
programs.singularity.enable = true;
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
services.flannel.iface = "eno2";
networking = {
useNetworkd = true;
hostName = name;
firewall = {
checkReversePath = "loose";
trustedInterfaces = [ "tailscale0" ];
allowedUDPPorts = [
41641
53
];
allowedTCPPorts = [
6443
4725
53
];
extraCommands = ''
# needed for nodeport access on k1 and k2
# iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
# iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -j SNAT --to-source 10.255.242.3
'';
};
};
systemd.network = {
networks = {
eno1 = {
DHCP = "no";
matchConfig.Name = "eno1";
address = [ "10.255.242.3/24" ];
routes = [ { Gateway = "10.255.242.1"; } ];
};
eno2 = {
DHCP = "no";
matchConfig.Name = "eno2";
address = [ "${address}/24" ];
routes = [
{
Destination = "172.16.239.0/24";
Gateway = "10.255.241.210";
}
];
};
ens2f1np1 = {
DHCP = "no";
matchConfig.Name = "ens2f1np1";
address = [ "10.255.244.99/24" ];
};
ibs2f0 = {
DHCP = "no";
matchConfig.Name = "ibs2f0";
address = [ "${ipoib}/24" ];
};
};
};
services.resolved = {
# DNS=[::1]:53
extraConfig = ''
DNSStubListener=no
'';
};
fileSystems = {
"/exports/public" = {
device = "/srv/public";
options = [ "bind" ];
};
};
nix.extraOptions = ''
# secret-key-files = /etc/nix/ekman.key
'';
services.prometheus.alertmanager.configuration.global = {
smtp_smart_host = "oceanbox-io.mail.protection.outlook.com";
smtp_tls_config = {
ca_file = "/etc/ssl/certs/ca-bundle.crt";
cert_file = "/var/lib/secrets/ekman-lb.oceanbox.io.pem";
key_file = "/var/lib/secrets/ekman-lb.oceanbox.io-key.pem";
};
smtp_hello = "ekman.oceanbox.io";
smtp_from = "noreply@oceanbox.io";
};
security.pam = {
services.sshd.googleAuthenticator.enable = true;
loginLimits = [
{
domain = "@users";
item = "rss";
type = "hard";
value = 16000000;
}
{
domain = "@users";
item = "cpu";
type = "hard";
value = 180;
}
];
};
system.activationScripts = {
home-permissions.text = ''
chmod 755 /home/olean
chmod 755 /home/frankgaa
chmod 755 /home/jonas
chmod 755 /home/stig
chmod 755 /home/bast
chmod 755 /home/mrtz
chmod 755 /home/avle
chmod 755 /home/simenlk
chmod 755 /home/ole
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it
services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
PubkeyAuthOptions verify-required
'';
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
virtualisation.docker.enable = pkgs.lib.mkForce true;
security.acme = {
acceptTerms = true;
defaults.email = "acme@oceanbox.io";
defaults.dnsPropagationCheck = false;
certs."ekman-lb.oceanbox.io" = {
dnsProvider = "namecheap";
environmentFile = "/var/lib/secrets/namecheap.key";
dnsPropagationCheck = false;
postRun = ''
cp fullchain.pem /var/lib/secrets/ekman-lb.oceanbox.io.pem
cp key.pem /var/lib/secrets/ekman-lb.oceanbox.io-key.pem
chmod 644 /var/lib/secrets/ekman-lb.oceanbox.io.pem
chmod 640 /var/lib/secrets/ekman-lb.oceanbox.io-key.pem
'';
};
};
programs.msmtp = {
enable = true;
accounts = {
default = {
auth = false;
host = "oceanbox-io.mail.protection.outlook.com";
port = 25;
from = "noreply@ekman.oceanbox.io";
tls_starttls = true;
tls = true;
tls_trust_file = "/etc/ssl/certs/ca-bundle.crt";
tls_key_file = "/var/lib/secrets/ekman-lb.oceanbox.io-key.pem";
tls_cert_file = "/var/lib/secrets/ekman-lb.oceanbox.io.pem";
};
};
defaults = {
aliases = "/etc/aliases";
};
};
environment.etc = {
"aliases" = {
text = ''
root: jonas.juselius@oceanbox.io, moritz.jorg@oceanbox.io
'';
mode = "0644";
};
};
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "both"; # for exit-node usage
extraUpFlags = [
"--login-server=https://headscale.svc.oceanbox.io"
"--accept-dns=true"
"--accept-routes=false"
"--advertise-tags=tag:hpc"
];
};
services.networkd-dispatcher = {
enable = true;
rules = {
"tailscale-router" = {
onState = [ "routable" ];
script = ''
#!${pkgs.runtimeShell}
${pkgs.ethtool}/bin/ethtool -K eno1 rx-udp-gro-forwarding on
${pkgs.ethtool}/bin/ethtool -K eno1 rx-gro-list off
${pkgs.ethtool}/bin/ethtool -K eno1 tx-udp-segmentation on
exit 0
'';
};
'';
};
};
};
imports = [
./hardware-configuration.nix
../default.nix
../mounts.nix
../myvnc.nix
../../dns.nix
];
boot.kernelParams = [
"console=tty0"
"console=ttyS0,115200"
];
systemd.services."serial-getty@ttyS0" = {
enable = true;
wantedBy = [ "getty.target" ];
serviceConfig.Restart = "always";
};
imports = [
./hardware-configuration.nix
../default.nix
../mounts.nix
../myvnc.nix
../../dns.nix
];
}