oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: move /home and /opt to cephfs and tweak mounts

Browse Source
This commit is contained in:
Jonas Juselius
2025-09-23 12:11:53 +02:00
parent 8894339216
commit 82a5328d7f
5 changed files with 70 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
rossby/c0/default.nix
View File

@@ -11,26 +11,6 @@ let
compute = {
deployment.tags = [ "compute" "c0" "cluster" ];
fileSystems = {
"/users" = {
device = "172.16.239.222:/home";
fsType = "nfs4";
options = [
"soft"
"defaults"
"noauto"
"x-systemd.automount"
];
};
};
systemd.automounts = [
{
where = "/users";
wantedBy = [ "default.target" ];
}
];
};
mkCompute = host:
@@ -46,9 +26,9 @@ let
automount.enable = true;
users = true;
opt = true;
work = false;
data = false;
ceph = false;
work = true;
data = true;
ceph = true;
};
};

2
rossby/c0/nodes.nix
View File

@@ -4,7 +4,7 @@ let
map (n: (
rec {
idx = 110 + n;
iface = if n < 11 then "enp34s0np0" else "enp33s0np0";
iface = if n > 1 && n < 11 then "enp34s0np0" else "enp33s0np0";
name = "c0-${toString n}";
address = "172.16.239.${toString idx}";
ipoib = "10.1.6.${toString idx}";

68
rossby/login/default.nix
View File

@@ -23,11 +23,11 @@ in
mounts = {
rdma.enable = false;
automount.enable = true;
users = false;
opt = false;
work = false;
data = false;
ceph = false;
users = true;
opt = true;
work = true;
data = true;
ceph = true;
};
};
@@ -170,28 +170,8 @@ in
};
fileSystems = {
"/exports/home" = {
device = "/home";
options = [ "bind" ];
};
"/exports/opt/bin" = {
device = "/opt/bin";
options = [ "bind" ];
};
"/exports/opt/sif" = {
device = "/opt/sif";
options = [ "bind" ];
};
"/exports/opt/singularity" = {
device = "/opt/singularity";
options = [ "bind" ];
};
"/exports/nfs-provisioner" = {
device = "/vol/nfs-provisioner";
options = [ "bind" ];
};
"/users" = {
device = "/home";
"/home" = {
device = "/ceph/volumes/nfs/home";
options = [ "bind" ];
};
"/vol/local-storage/vol1" = {
@@ -202,8 +182,42 @@ in
device = "/vol/vol2";
options = [ "bind" ];
};
# "/exports/home" = {
# device = "/home";
# options = [ "bind" ];
# };
# "/exports/opt/bin" = {
# device = "/opt/bin";
# options = [ "bind" ];
# };
# "/exports/opt/sif" = {
# device = "/opt/sif";
# options = [ "bind" ];
# };
# "/exports/opt/singularity" = {
# device = "/opt/singularity";
# options = [ "bind" ];
# };
# "/exports/nfs-provisioner" = {
# device = "/vol/nfs-provisioner";
# options = [ "bind" ];
# };
# "/users" = {
# device = "/home";
# options = [ "bind" ];
# };
};
systemd.automounts = [
# {
# wantedBy = [ "multi-user.target" ];
# automountConfig = {
# TimeoutIdleSec = "600";
# };
# where = "/home";
# }
];
nix.extraOptions = ''
# secret-key-files = /etc/nix/rossby.key
'';

15
rossby/manage/default.nix
View File

@@ -47,7 +47,7 @@ in {
mounts = {
rdma.enable = false;
automount.enable = true;
users = true;
users = false;
opt = true;
work = true;
data = true;
@@ -320,6 +320,19 @@ in {
"--advertise-routes=172.16.239.0/24,172.16.238.0/24"
];
};
services.networkd-dispatcher = {
enable = true;
rules = {
"tailscale-router" = {
onState = [ "routable" ];
script = ''
#!${pkgs.runtimeShell}
${pkgs.ethtool}/bin/ethtool -K enp65s0np0 rx-udp-gro-forwarding on rx-gro-list off
exit 0
'';
};
};
};
imports = [
./hardware-configuration.nix

34
rossby/mounts.nix
View File

@@ -5,41 +5,29 @@ let
options =
[ "soft" "defaults" "vers=4.2" ] ++
(if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []);
(if cfg.automount.enable then [ "x-systemd.automount" ] else []);
users =
if cfg.users then {
"/users" = {
device = "172.16.239.222:/home";
fsType = "nfs4";
options = [
"soft"
"defaults"
"noauto"
"x-systemd.automount"
];
device = "/ceph/volumes/nfs/home";
options = [ "bind" ];
};
} else {};
opt =
let
server = "172.16.239.222";
in
if cfg.opt then {
"/opt/bin" = {
device = "${server}:/opt/bin";
fsType = "nfs4";
inherit options;
device = "/ceph/volumes/nfs/opt/bin";
options = [ "bind" ];
};
"/opt/sif" = {
device = "${server}:/opt/sif";
fsType = "nfs4";
inherit options;
device = "/ceph/volumes/nfs/opt/sif";
options = [ "bind" ];
};
"/opt/singularity" = {
device = "${server}:/opt/singularity";
fsType = "nfs4";
inherit options;
device = "/ceph/volumes/nfs/opt/singularity";
options = [ "bind" ];
};
} else {};
@@ -68,6 +56,7 @@ let
options = [
"mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789"
"_netdev"
"x-systemd.automount"
];
};
} else {};
@@ -100,7 +89,7 @@ let
mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789
log file = /tmp/ceph-$pid.log
[client.oceanbox]
key = replaceme
key = AQDb7sZokwnUAxAANjnzxX0p+W/FUtSYryuyqg==
[client.rbd]
key = replaceme
'';
@@ -113,7 +102,6 @@ in
{
options.cluster.mounts = {
rdma.enable = mkEnableOption "Enable NFS over RDMA";
gbe100.enable = mkEnableOption "Enable NFS over 100 GbE";
automount.enable = mkEnableOption "Enable NFS automounting";
users = mkEnableOption "Enable /users";
opt = mkEnableOption "Enable /opt";