feat: move /home and /opt to cephfs and tweak mounts

rossby/c0/default.nix

@@ -11,26 +11,6 @@ let
 
   compute = {
     deployment.tags = [ "compute" "c0" "cluster" ];
-
-    fileSystems = {
-      "/users" = {
-        device = "172.16.239.222:/home";
-        fsType = "nfs4";
-        options = [
-          "soft"
-          "defaults"
-          "noauto"
-          "x-systemd.automount"
-        ];
-      };
-    };
-
-    systemd.automounts = [
-      {
-        where = "/users";
-        wantedBy = [ "default.target" ];
-      }
-    ];
   };
 
   mkCompute = host:
@@ -46,9 +26,9 @@ let
           automount.enable = true;
           users = true;
           opt = true;
-          work = false;
+          work = true;
-          data = false;
+          data = true;
-          ceph = false;
+          ceph = true;
         };
       };
 

rossby/c0/nodes.nix

@@ -4,7 +4,7 @@ let
 map (n: (
   rec {
     idx = 110 + n;
-    iface = if n < 11 then "enp34s0np0" else "enp33s0np0";
+    iface = if n > 1 && n < 11 then "enp34s0np0" else "enp33s0np0";
     name = "c0-${toString n}";
     address = "172.16.239.${toString idx}";
     ipoib = "10.1.6.${toString idx}";

rossby/login/default.nix

@@ -23,11 +23,11 @@ in
       mounts = {
         rdma.enable = false;
         automount.enable = true;
-        users = false;
+        users = true;
-        opt = false;
+        opt = true;
-        work = false;
+        work = true;
-        data = false;
+        data = true;
-        ceph = false;
+        ceph = true;
       };
     };
 
@@ -170,28 +170,8 @@ in
     };
 
     fileSystems = {
-      "/exports/home" = {
+      "/home" = {
-        device = "/home";
+        device = "/ceph/volumes/nfs/home";
-        options = [ "bind" ];
-      };
-      "/exports/opt/bin" = {
-        device = "/opt/bin";
-        options = [ "bind" ];
-      };
-      "/exports/opt/sif" = {
-        device = "/opt/sif";
-        options = [ "bind" ];
-      };
-      "/exports/opt/singularity" = {
-        device = "/opt/singularity";
-        options = [ "bind" ];
-      };
-      "/exports/nfs-provisioner" = {
-        device = "/vol/nfs-provisioner";
-        options = [ "bind" ];
-      };
-      "/users" = {
-        device = "/home";
         options = [ "bind" ];
       };
       "/vol/local-storage/vol1" = {
@@ -202,8 +182,42 @@ in
         device = "/vol/vol2";
         options = [ "bind" ];
       };
+      # "/exports/home" = {
+      #   device = "/home";
+      #   options = [ "bind" ];
+      # };
+      # "/exports/opt/bin" = {
+      #   device = "/opt/bin";
+      #   options = [ "bind" ];
+      # };
+      # "/exports/opt/sif" = {
+      #   device = "/opt/sif";
+      #   options = [ "bind" ];
+      # };
+      # "/exports/opt/singularity" = {
+      #   device = "/opt/singularity";
+      #   options = [ "bind" ];
+      # };
+      # "/exports/nfs-provisioner" = {
+      #   device = "/vol/nfs-provisioner";
+      #   options = [ "bind" ];
+      # };
+      # "/users" = {
+      #   device = "/home";
+      #   options = [ "bind" ];
+      # };
     };
 
+    systemd.automounts = [
+        # {
+        #   wantedBy = [ "multi-user.target" ];
+        #   automountConfig = {
+        #     TimeoutIdleSec = "600";
+        #   };
+        #   where = "/home";
+        # }
+      ];
+
     nix.extraOptions = ''
       # secret-key-files = /etc/nix/rossby.key
     '';

rossby/manage/default.nix

@@ -47,7 +47,7 @@ in {
       mounts = {
         rdma.enable = false;
         automount.enable = true;
-        users = true;
+        users = false;
         opt = true;
         work = true;
         data = true;
@@ -320,6 +320,19 @@ in {
       "--advertise-routes=172.16.239.0/24,172.16.238.0/24"
     ];
   };
+  services.networkd-dispatcher = {
+    enable = true;
+    rules = {
+      "tailscale-router" = {
+        onState = [ "routable" ];
+        script = ''
+        #!${pkgs.runtimeShell}
+        ${pkgs.ethtool}/bin/ethtool -K enp65s0np0 rx-udp-gro-forwarding on rx-gro-list off
+        exit 0
+        '';
+      };
+    };
+  };
 
   imports = [
       ./hardware-configuration.nix

rossby/mounts.nix

@@ -5,41 +5,29 @@ let
 
   options =
     [ "soft" "defaults" "vers=4.2" ] ++
-    (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []);
+    (if cfg.automount.enable then [ "x-systemd.automount" ] else []);
 
   users =
     if cfg.users then {
       "/users" = {
-        device = "172.16.239.222:/home";
+          device = "/ceph/volumes/nfs/home";
-        fsType = "nfs4";
+          options = [ "bind"  ];
-        options = [
-          "soft"
-          "defaults"
-          "noauto"
-          "x-systemd.automount"
-        ];
       };
     } else {};
 
   opt =
-    let
-      server = "172.16.239.222";
-    in
     if cfg.opt then {
       "/opt/bin" = {
-        device = "${server}:/opt/bin";
+          device = "/ceph/volumes/nfs/opt/bin";
-        fsType = "nfs4";
+          options = [ "bind" ];
-        inherit options;
       };
       "/opt/sif" = {
-        device = "${server}:/opt/sif";
+          device = "/ceph/volumes/nfs/opt/sif";
-        fsType = "nfs4";
+          options = [ "bind" ];
-        inherit options;
       };
       "/opt/singularity" = {
-        device = "${server}:/opt/singularity";
+          device = "/ceph/volumes/nfs/opt/singularity";
-        fsType = "nfs4";
+          options = [ "bind" ];
-        inherit options;
       };
     } else {};
 
@@ -68,6 +56,7 @@ let
           options = [
             "mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789"
             "_netdev"
+            "x-systemd.automount"
           ];
         };
       } else {};
@@ -100,7 +89,7 @@ let
           mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789
           log file = /tmp/ceph-$pid.log
           [client.oceanbox]
-          key = replaceme
+          key = AQDb7sZokwnUAxAANjnzxX0p+W/FUtSYryuyqg==
           [client.rbd]
           key = replaceme
         '';
@@ -113,7 +102,6 @@ in
 {
   options.cluster.mounts = {
     rdma.enable = mkEnableOption "Enable NFS over RDMA";
-    gbe100.enable = mkEnableOption "Enable NFS over 100 GbE";
     automount.enable = mkEnableOption "Enable NFS automounting";
     users = mkEnableOption "Enable /users";
     opt = mkEnableOption "Enable /opt";