oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

working stokes setup

Browse Source
This commit is contained in:
Jonas Juselius
2022-08-19 14:59:44 +02:00
parent 2dbbaa1ddd
commit 2f1c75a2f0
5 changed files with 53 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
clusters/stokes/cluster.nix
View File

@@ -30,7 +30,7 @@ let
boot = {
loader.systemd-boot.enable = true;
loader.efi.canTouchEfiVariables = true;
kernelPackages = pkgs.linuxPackages_5_4;
# kernelPackages = pkgs.linuxPackages_5_4;
kernelModules = [ "ib_umad" "ib_ipoib" ];
kernelParams = [
"console=ttyS0,115200"
@@ -38,10 +38,6 @@ let
];
};
services.udev.extraRules = ''
KERNEL=="ibp59s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222"
'';
console = {
font = "Lat2-Terminus16";
keyMap = "us";
@@ -112,7 +108,7 @@ let
slurm = {
client = true;
mungeKey = ./munge.key;
mungeUid = 996; # hack
mungeUid = 997; # hack
controlMachine = "stokes";
nodeName = [
"c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
@@ -128,7 +124,7 @@ let
beegfs = {
mds0-0 = {
mgmtdHost = "mds0-0";
connAuthFile = "";
connAuthFile = "/etc/beegfs/connauthfile";
client = {
enable = true;
mountPoint = "/work";
@@ -178,6 +174,19 @@ let
'';
};
fileSystems = {
"/opt" = {
device = "10.1.63.80:/opt";
fsType = "nfs";
options = [ "soft" "rdma" "defaults" "vers=4.2" ];
};
"/data" = {
device = "10.1.63.80:/data";
fsType = "nfs";
options = [ "soft" "rdma" "defaults" "vers=4.2" ];
};
};
environment.variables = {};
systemd.services."serial-getty@ttyS0".enable = true;
@@ -198,17 +207,13 @@ let
};
};
deployment = {
deployment.targetHost = cfg.address;
};
i40efix = {
boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
kernel = pkgs.linuxPackages_5_4.kernel;
}; in {
extraModulePackages = [ kernelExtras.i40e2 ];
kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
};
# boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
# kernel = pkgs.linuxPackages_5_4.kernel;
# }; in {
# extraModulePackages = [ kernelExtras.i40e2 ];
# kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
# };
};
shosts = {
@@ -310,7 +315,7 @@ let
in {
options.node = {
i40efix = mkEnableOption "Apply fix for i40e driver";
# i40efix = mkEnableOption "Apply fix for i40e driver";
myvnc = mkEnableOption "Enable myvnc script";
};
@@ -318,11 +323,11 @@ in {
config = mkMerge [
configuration
deployment
# deployment
shosts
(mkIf config.node.i40efix i40efix)
# (mkIf config.node.i40efix i40efix)
(mkIf config.node.myvnc myvnc)
];

39
clusters/stokes/default.nix
View File

@@ -6,15 +6,7 @@ let
# }) {};
pkgs = import <nixpkgs> {};
nodes =
with builtins;
let nodes = genList (n: n + 1) 8; in
map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;
stokes = import ./frontend.nix {
inherit pkgs;
inherit nodes;
};
nodes = import ./nodes.nix;
compute = {
# deployment.tags = [ "compute" ];
@@ -22,7 +14,7 @@ let
fileSystems = {
"/stokes" = {
device = "10.1.63.100:/home";
fsType = "nfs";
fsType = "nfs4";
options = [
"soft"
"defaults"
@@ -30,16 +22,6 @@ let
"x-systemd.automount"
];
};
"/opt" = {
device = "10.1.63.80:/opt";
fsType = "nfs";
options = [ "soft" "rdma" "defaults" ];
};
"/data" = {
device = "10.1.63.80:/data";
fsType = "nfs";
options = [ "soft" "rdma" "defaults" ];
};
};
systemd.automounts = [
@@ -63,8 +45,14 @@ let
# k8s = { inherit etcdCluster; };
};
deployment.targetHost = host.address;
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222"
# '';
node = {
i40efix = true;
# i40efix = true;
};
networking = {
@@ -82,7 +70,7 @@ let
} ];
};
interfaces."ibp65s0.2222" = {
interfaces.ibp65s0 = {
useDHCP = false;
ipv4.addresses = [ {
address = ipoib;
@@ -94,7 +82,8 @@ let
}
// compute;
};
in {
in builtins.foldl' (a: n: a // mkCompute n) {} nodes
#{
## morph
# network = {
# inherit pkgs;
@@ -104,6 +93,6 @@ in {
# };
# };
inherit stokes;
} // builtins.foldl' (a: n: a // mkCompute n) {} nodes
# inherit stokes;
#} // builtins.foldl' (a: n: a // mkCompute n) {} nodes

22
clusters/stokes/frontend.nix
View File

@@ -1,4 +1,7 @@
{ pkgs, nodes, ...}:
{ pkgs, ...}:
let
nodes = import ./nodes.nix;
in
{
# deployment.tags = [ "frontend" ];
node.myvnc = true;
@@ -85,7 +88,12 @@
};
};
# services.udev.extraRules = ''
# KERNEL=="ibp59s0", SUBSYSTEM=="net", ATTR{create_child}:="0x2222"
# '';
networking = {
hostName = "stokes";
useDHCP = false;
interfaces.eno1 = {
useDHCP = false;
@@ -101,7 +109,7 @@
prefixLength = 24;
} ];
};
interfaces."ibp59s0.2222" = {
interfaces.ibp59s0 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.1.63.100";
@@ -124,16 +132,6 @@
device = "/home";
options = [ "bind" ];
};
"/opt" = {
device = "10.1.63.80:/opt";
fsType = "nfs";
options = [ "soft" "rdma" "defaults" ];
};
"/data" = {
device = "10.1.63.80:/data";
fsType = "nfs";
options = [ "soft" "rdma" "defaults" ];
};
"/vol/local-storage/vol1" = {
device = "/vol/vol1";
options = [ "bind" ];

2
clusters/stokes/kernel.nix
View File

@@ -40,7 +40,7 @@ in
{
i40e2 = i40e;
overlay = self: super: {
linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
# linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
};
}

3
clusters/stokes/nodes.nix Normal file
View File

@@ -0,0 +1,3 @@
with builtins;
let nodes = genList (n: n + 1) 8; in
map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes