oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

wip: convert ekman to new cluster sturcture (not complete)

Browse Source
This commit is contained in:
Jonas Juselius
2025-09-12 12:53:56 +02:00
parent 899a7f4338
commit ba5f1b8add
95 changed files with 150 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

358
ekman/login/default.nix Normal file
View File

@@ -0,0 +1,358 @@
{ pkgs ? import <nixpkgs> {} }:
let
name = "ekman";
address = "10.255.241.100";
in
{
ekman = { config, pkgs, ... }: with pkgs; {
deployment.tags = [ "ekman" "login" ];
deployment.targetHost = address;
system.autoUpgrade.enable = lib.mkForce false;
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
};
cluster = {
compute = true;
k8sNode = true;
mounts = {
rdma.enable = true;
automount.enable = true;
home = false;
opt = false;
work = true;
data = true;
backup = true;
ceph = true;
};
};
features = {
host = {
inherit name;
inherit address;
};
myvnc.enable = false;
os = {
externalInterface = "enp33s0f0np0";
nfs.enable = true;
nfs.exports = ''
/exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
hpc = {
slurm.server = false;
slurm.slurmrestd = false;
frontend = false;
login = true;
knem = false;
};
k8s = {
master.enable = false;
node.enable = true;
};
desktop.enable = false;
# server.enable = true;
monitoring = {
# server = {
# enable = false;
# scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
# defaultAlertReceiver = {
# email_configs = [
# { to = "jonas.juselius@oceanbox.io"; }
# ];
# };
# pageAlertReceiver = {
# webhook_configs = [
# {
# url = "https://prometheus-msteams.k2.itpartner.no/ekman";
# http_config = {
# tls_config = { insecure_skip_verify = true; };
# };
# }
# ];
# };
# };
# webUI.enable = false;
# webUI.acmeEmail = "innovasjon@itpartner.no";
# webUI.allow = [
# "10.1.2.0/24"
# "172.19.254.0/24"
# "172.19.255.0/24"
# ];
infiniband-exporter = {
enable = true;
nameMap = ''
0x248a07030029d2fc "frontend"
0x248a07030029d104 "ekman"
0x5aa2e1fffe1edfca "fs-work"
0x1c34da0300787798 "fs-backup"
0xe8ebd3030024981e "c0-1"
0xe8ebd3030024a21a "c0-2"
0xe8ebd30300249a3a "c0-3"
0xe8ebd30300248b9e "c0-4"
0xe8ebd30300248b86 "c0-5"
0xe8ebd3030024998a "c0-6"
0xe8ebd30300248b8e "c0-7"
0xe8ebd3030024999e "c0-8"
0xe8ebd30300248fca "c0-9"
0xe8ebd3030024a216 "c0-10"
0xe8ebd30300248b96 "c0-11"
0xe8ebd30300248b9a "c0-12"
0xe8ebd303002495d2 "c0-13"
0xe8ebd303002495e2 "c0-14"
0xe8ebd30300248f42 "c0-15"
0xe8ebd303002495e6 "c0-16"
0xe8ebd3030024a2a2 "c0-17"
0xe8ebd3030024a2ae "c0-18"
'';
};
slurm-exporter = {
enable = true;
port = 6080;
};
};
};
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
services.flannel.iface = "enp33s0f3np3";
networking = {
useDHCP = false;
hostName = name;
interfaces.enp33s0f3np3 = {
useDHCP = false;
ipv4.addresses = [ {
address = address;
prefixLength = 24;
} ];
# ipv4.routes = [
# {
# address = "10.255.244.0";
# prefixLength = 24;
# via = "10.255.241.99";
# }
# ];
};
interfaces."ibp65s0f0" = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.243.100";
prefixLength = 24;
} ];
};
interfaces."enp65s0f1np1" = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.244.100";
prefixLength = 24;
} ];
};
interfaces.enp33s0f0np0 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.242.2";
prefixLength = 24;
} ];
ipv4.routes = [
# {
# address = "10.1.8.0";
# prefixLength = 24;
# via = "10.255.242.1";
# }
# {
# address = "10.1.30.0";
# prefixLength = 24;
# via = "10.255.242.1";
# }
];
};
defaultGateway = "10.255.242.1";
firewall = {
allowedTCPPorts = [ 4443 ];
extraCommands = ''
# needed for nodeport access on k1 and k2
# iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.2
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';
};
};
fileSystems = {
"/exports/home" = {
device = "/home";
options = [ "bind" ];
};
"/exports/opt/bin" = {
device = "/opt/bin";
options = [ "bind" ];
};
"/exports/opt/sif" = {
device = "/opt/sif";
options = [ "bind" ];
};
"/exports/opt/singularity" = {
device = "/opt/singularity";
options = [ "bind" ];
};
"/exports/nfs-provisioner" = {
device = "/vol/nfs-provisioner";
options = [ "bind" ];
};
"/frontend" = {
device = "/home";
options = [ "bind" ];
};
"/vol/local-storage/vol1" = {
device = "/vol/vol1";
options = [ "bind" ];
};
"/vol/local-storage/vol2" = {
device = "/vol/vol2";
options = [ "bind" ];
};
};
nix.extraOptions = ''
secret-key-files = /etc/nix/ekman.key
'';
# services.xserver = {
# enable = false;
# enableCtrlAltBackspace = true;
# layout = "us";
# xkbVariant = "altgr-intl";
# xkbOptions = "eurosign:e";
# displayManager = {
# gdm.enable = false;
# job.logToFile = true;
# };
# # desktopManager.xfce.enable = true;
# };
services.prometheus.alertmanager.configuration.global = {
smtp_smarthost = "smtpgw.itpartner.no";
# smtp_auth_username = "utvikling";
# smtp_auth_password = "S0m3rp0m@de#21!";
smtp_hello = "ekman.oceanbox.io";
smtp_from = "noreply@ekman.oceanbox.io";
};
# services.nginx = {
# virtualHosts = {
# "ds.matnoc.regnekraft.io" = {
# forceSSL = true;
# enableACME = true;
# serverAliases = [];
# locations."/" = {
# proxyPass = "http://localhost:9088";
# proxyWebsockets = false;
# extraConfig = ''
# allow 10.1.2.0/24;
# allow 172.19.254.0/24;
# allow 172.19.255.0/24;
# deny all;
# '';
# };
# };
# };
# };
# services.gitlab-runner = {
# enable = true;
# extraPackages = with pkgs; [
# singularity
# ];
# concurrent = 4;
# services = {
# sif = {
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
# executor = "shell";
# tagList = [ "ekman" "sif" ];
# };
# };
# };
# security.sudo.extraConfig = ''
# gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
# '';
security.pam = {
services.sshd.googleAuthenticator.enable = true;
loginLimits = [
{
domain = "@users";
item = "rss";
type = "hard";
value = 16000000;
}
{
domain = "@users";
item = "cpu";
type = "hard";
value = 180;
}
];
};
system.activationScripts = {
home-permissions.text = ''
chmod 755 /home/olean
chmod 755 /home/frankgaa
chmod 755 /home/jonas
chmod 755 /home/mrtz
chmod 755 /home/avle
chmod 755 /home/stig
chmod 755 /home/bast
chmod 755 /home/simenlk
chmod 755 /work/kraken
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it
services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
PubkeyAuthOptions verify-required
'';
environment.systemPackages = [];
virtualisation.docker.enable = pkgs.lib.mkForce true;
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "server"; # for exit-node usage
extraUpFlags = [
"--login-server=https://headscale.svc.oceanbox.io"
"--accept-dns"
"--advertise-exit-node"
"--advertise-routes=10.255.241.241.0/24"
"--advertise-tags=tag:ekman"
];
};
imports = [
./hardware-configuration.nix
../cluster.nix
../mounts.nix
../myvnc.nix
];
};
}