oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: upgrade ekman to new cluster structure

Browse Source
This commit is contained in:
Jonas Juselius
2025-09-23 13:40:16 +02:00
parent 46473c88dd
commit 96f8215c52
11 changed files with 44 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
ekman/c0/default.nix
View File

@@ -12,25 +12,6 @@ let
compute = { compute = {
deployment.tags = [ "compute" "c0" ]; deployment.tags = [ "compute" "c0" ];
fileSystems = {
"/frontend" = {
device = "10.255.241.100:/home";
fsType = "nfs4";
options = [
"soft"
"defaults"
"noauto"
"x-systemd.automount"
];
};
};
systemd.automounts = [
{
where = "/frontend";
wantedBy = [ "default.target" ];
}
];
}; };
mkCompute = host: mkCompute = host:
@@ -44,7 +25,7 @@ let
mounts = { mounts = {
rdma.enable = true; rdma.enable = true;
automount.enable = true; automount.enable = true;
home = false; users = true;
opt = true; opt = true;
work = true; work = true;
data = false; data = false;
@@ -97,7 +78,7 @@ let
}; };
imports = [ imports = [
hw hw
../cluster.nix ../default.nix
../mounts.nix ../mounts.nix
#./kernel.nix #./kernel.nix
]; ];

28
ekman/c0x/default.nix
View File

@@ -11,26 +11,6 @@ let
compute = { compute = {
deployment.tags = [ "compute" "c0" ]; deployment.tags = [ "compute" "c0" ];
fileSystems = {
"/frontend" = {
device = "10.255.241.100:/home";
fsType = "nfs4";
options = [
"soft"
"defaults"
"noauto"
"x-systemd.automount"
];
};
};
systemd.automounts = [
{
where = "/frontend";
wantedBy = [ "default.target" ];
}
];
}; };
mkCompute = host: mkCompute = host:
@@ -44,11 +24,11 @@ let
mounts = { mounts = {
rdma.enable = true; rdma.enable = true;
automount.enable = true; automount.enable = true;
home = false; users = true;
opt = true; opt = true;
work = true; work = true;
data = true; data = false;
backup = true; backup = false;
ceph = false; ceph = false;
}; };
}; };
@@ -97,7 +77,7 @@ let
}; };
imports = [ imports = [
hw hw
../cluster.nix ../default.nix
../mounts.nix ../mounts.nix
# ./kernel.nix # ./kernel.nix
]; ];

13
ekman/c1/default.nix
View File

@@ -10,13 +10,6 @@ let
nodes = import ./nodes.nix; nodes = import ./nodes.nix;
compute = { compute = {
deployment.tags = [ "compute" "c1" ]; deployment.tags = [ "compute" "c1" ];
systemd.automounts = [
{
where = "/frontend";
wantedBy = [ "default.target" ];
}
];
}; };
mkCompute = host: mkCompute = host:
@@ -31,10 +24,10 @@ let
rdma.enable = false; rdma.enable = false;
gbe100.enable = true; gbe100.enable = true;
automount.enable = true; automount.enable = true;
home = true; users = true;
opt = true; opt = true;
work = true; work = true;
data = false; data = true;
backup = false; backup = false;
ceph = true; ceph = true;
}; };
@@ -109,7 +102,7 @@ let
}; };
imports = [ imports = [
hw hw
../cluster.nix ../default.nix
../mounts.nix ../mounts.nix
]; ];
} }

16
ekman/default.nix
View File

@@ -96,7 +96,7 @@ let
}; };
cachix.enable = false; cachix.enable = false;
monitoring.nodeExporter.enable = false; monitoring.nodeExporter.enable = false;
hpc.mft.enable = false; # Mellanox MFT mft.enable = false; # Mellanox MFT
}; };
networking = { networking = {
@@ -250,7 +250,7 @@ let
hostNames = [ hostNames = [
"ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100" "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100"
]; ];
publicKeyFile = ./logon/ssh_host_key.pub; publicKeyFile = ./login/ssh_host_key.pub;
}; };
fs-work = { fs-work = {
hostNames = [ hostNames = [
@@ -293,6 +293,18 @@ let
permissions = "u+rs,g+rx,o+rx"; permissions = "u+rs,g+rx,o+rx";
}; };
}; };
# Use nvd to get package diff before apply
system.activationScripts.system-diff = {
supportsDryActivation = true; # safe: only outputs to stdout
text = ''
export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
if [ -e /run/current-system ]; then
${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
fi
'';
};
}; };
openssh-shosts = pkgs.openssh.overrideAttrs (attrs: { openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {

4
ekman/fs-backup/default.nix
View File

@@ -60,7 +60,7 @@ in {
mounts = { mounts = {
rdma.enable = false; rdma.enable = false;
automount.enable = true; automount.enable = true;
home = false; users = false;
opt = false; opt = false;
work = false; work = false;
data = false; data = false;
@@ -172,7 +172,7 @@ in {
imports = [ imports = [
./hardware-configuration.nix ./hardware-configuration.nix
../cluster.nix ../default.nix
../mounts.nix ../mounts.nix
]; ];
}; };

4
ekman/fs-work/default.nix
View File

@@ -37,7 +37,7 @@ in {
mounts = { mounts = {
rdma.enable = true; rdma.enable = true;
automount.enable = true; automount.enable = true;
home = true; users = true;
opt = false; opt = false;
work = false; work = false;
data = false; data = false;
@@ -165,7 +165,7 @@ in {
imports = [ imports = [
./hardware-configuration.nix ./hardware-configuration.nix
../cluster.nix ../default.nix
../mounts.nix ../mounts.nix
]; ];
}; };

16
ekman/hive.nix
View File

@@ -12,14 +12,14 @@ let
allowLocalDeployment = true; allowLocalDeployment = true;
targetHost = null; targetHost = null;
}; };
imports = [ ./ekman/manage ]; imports = [ ./manage ];
}; };
ekman-login = import ./ekman/login { inherit pkgs; }; login = import ./login { inherit pkgs; };
c0 = import ./ekman/c0 { inherit pkgs; }; c0 = import ./c0 { inherit pkgs; };
c0x = import ./ekman/c0x { inherit pkgs; }; c0x = import ./c0x { inherit pkgs; };
c1 = import ./ekman/c1 { inherit pkgs; }; c1 = import ./c1 { inherit pkgs; };
fs-work = import ./ekman/fs-work { inherit pkgs; }; fs-work = import ./fs-work { inherit pkgs; };
fs-backup = import ./ekman/fs-backup { inherit pkgs; }; fs-backup = import ./fs-backup { inherit pkgs; };
in in
{ inherit ekman-manage; } // ekman-login // c0 // c0x // c1 // fs-work // fs-backup { inherit ekman-manage; frontend = ekman-manage; } // login // c0 // c0x // c1 // fs-work // fs-backup

2
ekman/hosts.nix
View File

@@ -11,7 +11,7 @@
10.255.241.100 etcd0 etcd0.cluster.local 10.255.241.100 etcd0 etcd0.cluster.local
10.255.241.80 etcd1 etcd1.cluster.local 10.255.241.80 etcd1 etcd1.cluster.local
10.255.241.90 etcd2 etcd2.cluster.local 10.255.241.90 etcd2 etcd2.cluster.local
10.255.241.99 ekman-manage frontend frontend.cluster.local frontend.cluster.local 10.255.241.99 ekman-manage frontend ekman-manage.cluster.local frontend.cluster.local
10.255.243.99 ibfrontend ibfrontend.cluster.local ibfrontend.cluster.local 10.255.243.99 ibfrontend ibfrontend.cluster.local ibfrontend.cluster.local
10.255.241.101 c0-1 c0-1.cluster.local 10.255.241.101 c0-1 c0-1.cluster.local

10
ekman/login/default.nix
View File

@@ -22,7 +22,7 @@ in
mounts = { mounts = {
rdma.enable = true; rdma.enable = true;
automount.enable = true; automount.enable = true;
home = false; users = false;
opt = false; opt = false;
work = true; work = true;
data = true; data = true;
@@ -51,7 +51,7 @@ in
hpc = { hpc = {
slurm.server = false; slurm.server = false;
slurm.slurmrestd = false; slurm.slurmrestd = false;
frontend = false; manage = false;
login = true; login = true;
knem = false; knem = false;
}; };
@@ -193,7 +193,7 @@ in
}; };
fileSystems = { fileSystems = {
"/exports/home" = { "/exports/users" = {
device = "/home"; device = "/home";
options = [ "bind" ]; options = [ "bind" ];
}; };
@@ -213,7 +213,7 @@ in
device = "/vol/nfs-provisioner"; device = "/vol/nfs-provisioner";
options = [ "bind" ]; options = [ "bind" ];
}; };
"/frontend" = { "/users" = {
device = "/home"; device = "/home";
options = [ "bind" ]; options = [ "bind" ];
}; };
@@ -349,7 +349,7 @@ in
imports = [ imports = [
./hardware-configuration.nix ./hardware-configuration.nix
../cluster.nix ../default.nix
../mounts.nix ../mounts.nix
../myvnc.nix ../myvnc.nix
]; ];

15
ekman/manage/default.nix
View File

@@ -55,7 +55,7 @@ in {
mounts = { mounts = {
rdma.enable = true; rdma.enable = true;
automount.enable = true; automount.enable = true;
home = true; users = true;
opt = true; opt = true;
work = true; work = true;
data = true; data = true;
@@ -88,7 +88,7 @@ in {
slurm.server = true; slurm.server = true;
slurm.slurmrestd = false; slurm.slurmrestd = false;
slurm.mungeUid = 996; slurm.mungeUid = 996;
frontend = true; manage = true;
}; };
k8s = { k8s = {
@@ -283,17 +283,6 @@ in {
''; '';
}; };
# Use nvd to get package diff before apply
system.activationScripts.system-diff = {
supportsDryActivation = true; # safe: only outputs to stdout
text = ''
export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
if [ -e /run/current-system ]; then
${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
fi
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it # ssh-rsa is deprecated, but putty/winscp users use it
services.openssh.extraConfig = '' services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256

2
ekman/mounts.nix
View File

@@ -15,7 +15,7 @@ let
users = users =
if cfg.users then { if cfg.users then {
"/users" = { "/users" = {
device = "10.255.241.100:/home"; device = "10.255.241.100:/users";
fsType = "nfs4"; fsType = "nfs4";
options = [ options = [
"soft" "soft"