From 82a5328d7fdf3b3157def20cb2110dcc89c85c0e Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Tue, 23 Sep 2025 12:11:53 +0200 Subject: [PATCH] feat: move /home and /opt to cephfs and tweak mounts --- rossby/c0/default.nix | 26 ++------------- rossby/c0/nodes.nix | 2 +- rossby/login/default.nix | 68 +++++++++++++++++++++++---------------- rossby/manage/default.nix | 15 ++++++++- rossby/mounts.nix | 34 +++++++------------- 5 files changed, 70 insertions(+), 75 deletions(-) diff --git a/rossby/c0/default.nix b/rossby/c0/default.nix index b0866df..613bc77 100644 --- a/rossby/c0/default.nix +++ b/rossby/c0/default.nix @@ -11,26 +11,6 @@ let compute = { deployment.tags = [ "compute" "c0" "cluster" ]; - - fileSystems = { - "/users" = { - device = "172.16.239.222:/home"; - fsType = "nfs4"; - options = [ - "soft" - "defaults" - "noauto" - "x-systemd.automount" - ]; - }; - }; - - systemd.automounts = [ - { - where = "/users"; - wantedBy = [ "default.target" ]; - } - ]; }; mkCompute = host: @@ -46,9 +26,9 @@ let automount.enable = true; users = true; opt = true; - work = false; - data = false; - ceph = false; + work = true; + data = true; + ceph = true; }; }; diff --git a/rossby/c0/nodes.nix b/rossby/c0/nodes.nix index a9a9eb1..fb745d0 100644 --- a/rossby/c0/nodes.nix +++ b/rossby/c0/nodes.nix @@ -4,7 +4,7 @@ let map (n: ( rec { idx = 110 + n; - iface = if n < 11 then "enp34s0np0" else "enp33s0np0"; + iface = if n > 1 && n < 11 then "enp34s0np0" else "enp33s0np0"; name = "c0-${toString n}"; address = "172.16.239.${toString idx}"; ipoib = "10.1.6.${toString idx}"; diff --git a/rossby/login/default.nix b/rossby/login/default.nix index 76fade5..a73530d 100644 --- a/rossby/login/default.nix +++ b/rossby/login/default.nix @@ -23,11 +23,11 @@ in mounts = { rdma.enable = false; automount.enable = true; - users = false; - opt = false; - work = false; - data = false; - ceph = false; + users = true; + opt = true; + work = true; + data = true; + ceph = true; }; }; @@ -170,28 +170,8 @@ in }; fileSystems = { - "/exports/home" = { - device = "/home"; - options = [ "bind" ]; - }; - "/exports/opt/bin" = { - device = "/opt/bin"; - options = [ "bind" ]; - }; - "/exports/opt/sif" = { - device = "/opt/sif"; - options = [ "bind" ]; - }; - "/exports/opt/singularity" = { - device = "/opt/singularity"; - options = [ "bind" ]; - }; - "/exports/nfs-provisioner" = { - device = "/vol/nfs-provisioner"; - options = [ "bind" ]; - }; - "/users" = { - device = "/home"; + "/home" = { + device = "/ceph/volumes/nfs/home"; options = [ "bind" ]; }; "/vol/local-storage/vol1" = { @@ -202,8 +182,42 @@ in device = "/vol/vol2"; options = [ "bind" ]; }; + # "/exports/home" = { + # device = "/home"; + # options = [ "bind" ]; + # }; + # "/exports/opt/bin" = { + # device = "/opt/bin"; + # options = [ "bind" ]; + # }; + # "/exports/opt/sif" = { + # device = "/opt/sif"; + # options = [ "bind" ]; + # }; + # "/exports/opt/singularity" = { + # device = "/opt/singularity"; + # options = [ "bind" ]; + # }; + # "/exports/nfs-provisioner" = { + # device = "/vol/nfs-provisioner"; + # options = [ "bind" ]; + # }; + # "/users" = { + # device = "/home"; + # options = [ "bind" ]; + # }; }; + systemd.automounts = [ + # { + # wantedBy = [ "multi-user.target" ]; + # automountConfig = { + # TimeoutIdleSec = "600"; + # }; + # where = "/home"; + # } + ]; + nix.extraOptions = '' # secret-key-files = /etc/nix/rossby.key ''; diff --git a/rossby/manage/default.nix b/rossby/manage/default.nix index 46afac0..dd2159e 100644 --- a/rossby/manage/default.nix +++ b/rossby/manage/default.nix @@ -47,7 +47,7 @@ in { mounts = { rdma.enable = false; automount.enable = true; - users = true; + users = false; opt = true; work = true; data = true; @@ -320,6 +320,19 @@ in { "--advertise-routes=172.16.239.0/24,172.16.238.0/24" ]; }; + services.networkd-dispatcher = { + enable = true; + rules = { + "tailscale-router" = { + onState = [ "routable" ]; + script = '' + #!${pkgs.runtimeShell} + ${pkgs.ethtool}/bin/ethtool -K enp65s0np0 rx-udp-gro-forwarding on rx-gro-list off + exit 0 + ''; + }; + }; + }; imports = [ ./hardware-configuration.nix diff --git a/rossby/mounts.nix b/rossby/mounts.nix index c980e50..3db2dc7 100644 --- a/rossby/mounts.nix +++ b/rossby/mounts.nix @@ -5,41 +5,29 @@ let options = [ "soft" "defaults" "vers=4.2" ] ++ - (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []); + (if cfg.automount.enable then [ "x-systemd.automount" ] else []); users = if cfg.users then { "/users" = { - device = "172.16.239.222:/home"; - fsType = "nfs4"; - options = [ - "soft" - "defaults" - "noauto" - "x-systemd.automount" - ]; + device = "/ceph/volumes/nfs/home"; + options = [ "bind" ]; }; } else {}; opt = - let - server = "172.16.239.222"; - in if cfg.opt then { "/opt/bin" = { - device = "${server}:/opt/bin"; - fsType = "nfs4"; - inherit options; + device = "/ceph/volumes/nfs/opt/bin"; + options = [ "bind" ]; }; "/opt/sif" = { - device = "${server}:/opt/sif"; - fsType = "nfs4"; - inherit options; + device = "/ceph/volumes/nfs/opt/sif"; + options = [ "bind" ]; }; "/opt/singularity" = { - device = "${server}:/opt/singularity"; - fsType = "nfs4"; - inherit options; + device = "/ceph/volumes/nfs/opt/singularity"; + options = [ "bind" ]; }; } else {}; @@ -68,6 +56,7 @@ let options = [ "mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789" "_netdev" + "x-systemd.automount" ]; }; } else {}; @@ -100,7 +89,7 @@ let mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789 log file = /tmp/ceph-$pid.log [client.oceanbox] - key = replaceme + key = AQDb7sZokwnUAxAANjnzxX0p+W/FUtSYryuyqg== [client.rbd] key = replaceme ''; @@ -113,7 +102,6 @@ in { options.cluster.mounts = { rdma.enable = mkEnableOption "Enable NFS over RDMA"; - gbe100.enable = mkEnableOption "Enable NFS over 100 GbE"; automount.enable = mkEnableOption "Enable NFS automounting"; users = mkEnableOption "Enable /users"; opt = mkEnableOption "Enable /opt";