working stokes setup

clusters/stokes/cluster.nix

@@ -30,7 +30,7 @@ let
     boot = {
       loader.systemd-boot.enable = true;
       loader.efi.canTouchEfiVariables = true;
-      kernelPackages = pkgs.linuxPackages_5_4;
+      # kernelPackages = pkgs.linuxPackages_5_4;
       kernelModules = [ "ib_umad" "ib_ipoib" ];
       kernelParams = [
         "console=ttyS0,115200"
@@ -38,10 +38,6 @@ let
       ];
     };
 
-    services.udev.extraRules = ''
-      KERNEL=="ibp59s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222"
-    '';
-
     console = {
       font = "Lat2-Terminus16";
       keyMap = "us";
@@ -112,7 +108,7 @@ let
         slurm = {
           client = true;
           mungeKey = ./munge.key;
-          mungeUid = 996; # hack
+          mungeUid = 997; # hack
           controlMachine = "stokes";
           nodeName = [
             "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
@@ -128,7 +124,7 @@ let
           beegfs = {
             mds0-0 = {
               mgmtdHost = "mds0-0";
-              connAuthFile = "";
+              connAuthFile = "/etc/beegfs/connauthfile";
               client = {
                 enable = true;
                 mountPoint = "/work";
@@ -178,6 +174,19 @@ let
       '';
     };
 
+    fileSystems = {
+      "/opt" = {
+        device = "10.1.63.80:/opt";
+        fsType = "nfs";
+        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
+      };
+      "/data" = {
+        device = "10.1.63.80:/data";
+        fsType = "nfs";
+        options = [ "soft" "rdma" "defaults" "vers=4.2" ];
+      };
+    };
+
     environment.variables = {};
 
     systemd.services."serial-getty@ttyS0".enable = true;
@@ -198,17 +207,13 @@ let
     };
   };
 
-  deployment = {
-    deployment.targetHost = cfg.address;
-  };
-
   i40efix = {
-    boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
+    # boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
-      kernel = pkgs.linuxPackages_5_4.kernel;
+    #   kernel = pkgs.linuxPackages_5_4.kernel;
-    }; in {
+    # }; in {
-      extraModulePackages = [ kernelExtras.i40e2 ];
+    #   extraModulePackages = [ kernelExtras.i40e2 ];
-      kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
+    #   kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
-    };
+    # };
   };
 
   shosts = {
@@ -310,7 +315,7 @@ let
 
 in {
   options.node = {
-    i40efix = mkEnableOption "Apply fix for i40e driver";
+    # i40efix = mkEnableOption "Apply fix for i40e driver";
 
     myvnc = mkEnableOption "Enable myvnc script";
   };
@@ -318,11 +323,11 @@ in {
   config = mkMerge [
     configuration
 
-    deployment
+    # deployment
 
     shosts
 
-    (mkIf config.node.i40efix i40efix)
+    # (mkIf config.node.i40efix i40efix)
 
     (mkIf config.node.myvnc myvnc)
   ];

clusters/stokes/default.nix

@@ -6,15 +6,7 @@ let
   # }) {};
   pkgs = import <nixpkgs> {};
 
-  nodes =
+  nodes = import ./nodes.nix;
-    with builtins;
-    let nodes = genList (n: n + 1) 8; in
-    map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;
-
-  stokes = import ./frontend.nix {
-    inherit pkgs;
-    inherit nodes;
-  };
 
   compute = {
     # deployment.tags = [ "compute" ];
@@ -22,7 +14,7 @@ let
     fileSystems = {
       "/stokes" = {
         device = "10.1.63.100:/home";
-        fsType = "nfs";
+        fsType = "nfs4";
         options = [
           "soft"
           "defaults"
@@ -30,16 +22,6 @@ let
           "x-systemd.automount"
         ];
       };
-      "/opt" = {
-        device = "10.1.63.80:/opt";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" ];
-      };
-      "/data" = {
-        device = "10.1.63.80:/data";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" ];
-      };
     };
 
     systemd.automounts = [
@@ -63,8 +45,14 @@ let
         # k8s = { inherit etcdCluster; };
       };
 
+      deployment.targetHost = host.address;
+
+      # services.udev.extraRules = ''
+      #   KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222"
+      # '';
+
       node = {
-        i40efix = true;
+        # i40efix = true;
       };
 
       networking = {
@@ -82,7 +70,7 @@ let
           } ];
 
         };
-        interfaces."ibp65s0.2222" = {
+        interfaces.ibp65s0 = {
           useDHCP = false;
           ipv4.addresses = [ {
             address = ipoib;
@@ -94,7 +82,8 @@ let
     }
     // compute;
 };
-in {
+in builtins.foldl' (a: n: a // mkCompute n) {} nodes
+#{
     ## morph
     # network =  {
     #   inherit pkgs;
@@ -104,6 +93,6 @@ in {
     #   };
     # };
 
-    inherit stokes;
+    # inherit stokes;
-} // builtins.foldl' (a: n: a // mkCompute n) {} nodes
+#} // builtins.foldl' (a: n: a // mkCompute n) {} nodes
 

clusters/stokes/frontend.nix

@@ -1,4 +1,7 @@
-{ pkgs, nodes, ...}:
+{ pkgs, ...}:
+let
+  nodes = import ./nodes.nix;
+in
 {
     # deployment.tags = [ "frontend" ];
     node.myvnc = true;
@@ -85,7 +88,12 @@
       };
     };
 
+    # services.udev.extraRules = ''
+    #   KERNEL=="ibp59s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222"
+    # '';
+
     networking = {
+      hostName = "stokes";
       useDHCP = false;
       interfaces.eno1 = {
         useDHCP = false;
@@ -101,7 +109,7 @@
           prefixLength = 24;
         } ];
       };
-      interfaces."ibp59s0.2222" = {
+      interfaces.ibp59s0 = {
         useDHCP = false;
         ipv4.addresses = [ {
           address = "10.1.63.100";
@@ -124,16 +132,6 @@
         device = "/home";
         options = [ "bind" ];
       };
-      "/opt" = {
-        device = "10.1.63.80:/opt";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" ];
-      };
-      "/data" = {
-        device = "10.1.63.80:/data";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" ];
-      };
       "/vol/local-storage/vol1" = {
         device = "/vol/vol1";
         options = [ "bind" ];

clusters/stokes/kernel.nix

@@ -40,7 +40,7 @@ in
 {
   i40e2 = i40e;
   overlay = self: super: {
-    linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
+    # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
   };
 }
 

clusters/stokes/nodes.nix

@@ -0,0 +1,3 @@
+with builtins;
+let nodes = genList (n: n + 1) 8; in
+map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes