diff --git a/clusters/stokes/cluster.nix b/clusters/stokes/cluster.nix index 125f056..c05f156 100644 --- a/clusters/stokes/cluster.nix +++ b/clusters/stokes/cluster.nix @@ -30,7 +30,7 @@ let boot = { loader.systemd-boot.enable = true; loader.efi.canTouchEfiVariables = true; - kernelPackages = pkgs.linuxPackages_5_4; + # kernelPackages = pkgs.linuxPackages_5_4; kernelModules = [ "ib_umad" "ib_ipoib" ]; kernelParams = [ "console=ttyS0,115200" @@ -38,10 +38,6 @@ let ]; }; - services.udev.extraRules = '' - KERNEL=="ibp59s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222" - ''; - console = { font = "Lat2-Terminus16"; keyMap = "us"; @@ -112,7 +108,7 @@ let slurm = { client = true; mungeKey = ./munge.key; - mungeUid = 996; # hack + mungeUid = 997; # hack controlMachine = "stokes"; nodeName = [ "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" @@ -128,7 +124,7 @@ let beegfs = { mds0-0 = { mgmtdHost = "mds0-0"; - connAuthFile = ""; + connAuthFile = "/etc/beegfs/connauthfile"; client = { enable = true; mountPoint = "/work"; @@ -178,6 +174,19 @@ let ''; }; + fileSystems = { + "/opt" = { + device = "10.1.63.80:/opt"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; + }; + "/data" = { + device = "10.1.63.80:/data"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; + }; + }; + environment.variables = {}; systemd.services."serial-getty@ttyS0".enable = true; @@ -198,17 +207,13 @@ let }; }; - deployment = { - deployment.targetHost = cfg.address; - }; - i40efix = { - boot = let kernelExtras = pkgs.callPackage ./kernel.nix { - kernel = pkgs.linuxPackages_5_4.kernel; - }; in { - extraModulePackages = [ kernelExtras.i40e2 ]; - kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ]; - }; + # boot = let kernelExtras = pkgs.callPackage ./kernel.nix { + # kernel = pkgs.linuxPackages_5_4.kernel; + # }; in { + # extraModulePackages = [ kernelExtras.i40e2 ]; + # kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ]; + # }; }; shosts = { @@ -310,7 +315,7 @@ let in { options.node = { - i40efix = mkEnableOption "Apply fix for i40e driver"; + # i40efix = mkEnableOption "Apply fix for i40e driver"; myvnc = mkEnableOption "Enable myvnc script"; }; @@ -318,11 +323,11 @@ in { config = mkMerge [ configuration - deployment + # deployment shosts - (mkIf config.node.i40efix i40efix) + # (mkIf config.node.i40efix i40efix) (mkIf config.node.myvnc myvnc) ]; diff --git a/clusters/stokes/default.nix b/clusters/stokes/default.nix index 4309bd7..49e85cd 100644 --- a/clusters/stokes/default.nix +++ b/clusters/stokes/default.nix @@ -6,15 +6,7 @@ let # }) {}; pkgs = import {}; - nodes = - with builtins; - let nodes = genList (n: n + 1) 8; in - map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes; - - stokes = import ./frontend.nix { - inherit pkgs; - inherit nodes; - }; + nodes = import ./nodes.nix; compute = { # deployment.tags = [ "compute" ]; @@ -22,7 +14,7 @@ let fileSystems = { "/stokes" = { device = "10.1.63.100:/home"; - fsType = "nfs"; + fsType = "nfs4"; options = [ "soft" "defaults" @@ -30,16 +22,6 @@ let "x-systemd.automount" ]; }; - "/opt" = { - device = "10.1.63.80:/opt"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" ]; - }; - "/data" = { - device = "10.1.63.80:/data"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" ]; - }; }; systemd.automounts = [ @@ -63,8 +45,14 @@ let # k8s = { inherit etcdCluster; }; }; + deployment.targetHost = host.address; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222" + # ''; + node = { - i40efix = true; + # i40efix = true; }; networking = { @@ -82,7 +70,7 @@ let } ]; }; - interfaces."ibp65s0.2222" = { + interfaces.ibp65s0 = { useDHCP = false; ipv4.addresses = [ { address = ipoib; @@ -94,7 +82,8 @@ let } // compute; }; -in { +in builtins.foldl' (a: n: a // mkCompute n) {} nodes +#{ ## morph # network = { # inherit pkgs; @@ -104,6 +93,6 @@ in { # }; # }; - inherit stokes; -} // builtins.foldl' (a: n: a // mkCompute n) {} nodes + # inherit stokes; +#} // builtins.foldl' (a: n: a // mkCompute n) {} nodes diff --git a/clusters/stokes/frontend.nix b/clusters/stokes/frontend.nix index 471f305..3bb4511 100644 --- a/clusters/stokes/frontend.nix +++ b/clusters/stokes/frontend.nix @@ -1,4 +1,7 @@ -{ pkgs, nodes, ...}: +{ pkgs, ...}: +let + nodes = import ./nodes.nix; +in { # deployment.tags = [ "frontend" ]; node.myvnc = true; @@ -85,7 +88,12 @@ }; }; + # services.udev.extraRules = '' + # KERNEL=="ibp59s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222" + # ''; + networking = { + hostName = "stokes"; useDHCP = false; interfaces.eno1 = { useDHCP = false; @@ -101,7 +109,7 @@ prefixLength = 24; } ]; }; - interfaces."ibp59s0.2222" = { + interfaces.ibp59s0 = { useDHCP = false; ipv4.addresses = [ { address = "10.1.63.100"; @@ -124,16 +132,6 @@ device = "/home"; options = [ "bind" ]; }; - "/opt" = { - device = "10.1.63.80:/opt"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" ]; - }; - "/data" = { - device = "10.1.63.80:/data"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" ]; - }; "/vol/local-storage/vol1" = { device = "/vol/vol1"; options = [ "bind" ]; diff --git a/clusters/stokes/kernel.nix b/clusters/stokes/kernel.nix index f865667..750da0d 100644 --- a/clusters/stokes/kernel.nix +++ b/clusters/stokes/kernel.nix @@ -40,7 +40,7 @@ in { i40e2 = i40e; overlay = self: super: { - linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; + # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; }; } diff --git a/clusters/stokes/nodes.nix b/clusters/stokes/nodes.nix new file mode 100644 index 0000000..7d16834 --- /dev/null +++ b/clusters/stokes/nodes.nix @@ -0,0 +1,3 @@ +with builtins; +let nodes = genList (n: n + 1) 8; in +map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes