From 3ad0687026d159ce7d01b6cb30378599ff365713 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Wed, 8 May 2024 13:16:13 +0200 Subject: [PATCH] feat: refactor and unify network mounts throughout the cluster --- cluster/c0/default.nix | 16 ++++- cluster/c1/default.nix | 51 ++++++++++----- cluster/c1/hw/c1-2.nix | 1 + cluster/ceph.nix | 33 ---------- cluster/cluster.nix | 3 +- cluster/ekman/default.nix | 15 ++++- cluster/fs0/default.nix | 31 +++++----- cluster/fs1/default.nix | 68 +++++--------------- cluster/fs2/default.nix | 66 +++++--------------- cluster/mounts.nix | 127 ++++++++++++++++++++++++++++++++++++++ cluster/nfs.nix | 24 ------- configuration.nix | 36 ++++++++--- modules | 2 +- 13 files changed, 269 insertions(+), 204 deletions(-) delete mode 100644 cluster/ceph.nix create mode 100644 cluster/mounts.nix delete mode 100644 cluster/nfs.nix diff --git a/cluster/c0/default.nix b/cluster/c0/default.nix index 4dee0e6..c61e25e 100644 --- a/cluster/c0/default.nix +++ b/cluster/c0/default.nix @@ -41,6 +41,16 @@ let cluster = { compute = true; k8sNode = true; + mounts = { + rdma.enable = true; + automount.enable = false; + home = false; + opt = true; + work = true; + data = true; + backup = true; + ceph = false; + }; }; features = { @@ -84,7 +94,11 @@ let } ]; }; }; - imports = [ ../cluster.nix hw ]; + imports = [ + hw + ../cluster.nix + ../mounts.nix + ]; } // compute; }; diff --git a/cluster/c1/default.nix b/cluster/c1/default.nix index cb94fa9..590c574 100644 --- a/cluster/c1/default.nix +++ b/cluster/c1/default.nix @@ -8,23 +8,9 @@ let # pkgs = import {}; nodes = import ./nodes.nix; - compute = { deployment.tags = [ "compute" "c1" ]; - fileSystems = { - "/frontend" = { - device = "10.255.241.100:/home"; - fsType = "nfs4"; - options = [ - "soft" - "defaults" - "noauto" - "x-systemd.automount" - ]; - }; - }; - systemd.automounts = [ { where = "/frontend"; @@ -41,6 +27,16 @@ let cluster = { compute = true; k8sNode = true; + mounts = { + rdma.enable = if host.name == "c1-1" || host.name == "c1-2" then false else true; + automount.enable = false; + home = true; + opt = true; + work = true; + data = true; + backup = true; + ceph = if host.name == "c1-1" || host.name == "c1-2" then true else false; + }; }; features = { @@ -59,6 +55,21 @@ let # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" # ''; + boot.kernel.sysctl = { + "net.ipv4.tcp_timestamps" = 0; + "net.ipv4.tcp_sack" = 1; + "net.core.netdev_max_backlog" = 250000; + "net.core.rmem_max" = 4194304; + "net.core.wmem_max" = 4194304; + "net.core.rmem_default" = 4194304; + "net.core.wmem_default" = 4194304; + "net.core.optmem_max" = 4194304; + "net.ipv4.tcp_rmem" = "4096 87380 4194304"; + "net.ipv4.tcp_wmem" = "4096 65536 4194304"; + "net.ipv4.tcp_low_latency" = 1; + "net.ipv4.tcp_adv_win_scale" = 1; + }; + networking = { hostName = host.name; useDHCP = false; @@ -82,10 +93,18 @@ let prefixLength = 24; } ]; }; + interfaces.enp65s0np0 = { + useDHCP = false; + ipv4.addresses = [ { + address = builtins.replaceStrings [ "243" ] [ "244" ] host.ipoib; + prefixLength = 24; + } ]; + }; }; imports = [ - ../cluster.nix hw - ../ceph.nix + hw + ../cluster.nix + ../mounts.nix ]; } // compute; diff --git a/cluster/c1/hw/c1-2.nix b/cluster/c1/hw/c1-2.nix index 1f638eb..c97bce4 100644 --- a/cluster/c1/hw/c1-2.nix +++ b/cluster/c1/hw/c1-2.nix @@ -12,6 +12,7 @@ boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; + # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_7; fileSystems."/" = { device = "/dev/disk/by-uuid/d89e1496-fda1-4de0-b2cc-474967b04402"; diff --git a/cluster/ceph.nix b/cluster/ceph.nix deleted file mode 100644 index d926d03..0000000 --- a/cluster/ceph.nix +++ /dev/null @@ -1,33 +0,0 @@ -{ - fileSystems = { - "/ceph" = { - device = "10.255.241.30/10.255.241.31/10.255.241.32:6789:/"; - fsType = "ceph"; - options = [ "name=oceanbox" "secretfile=/etc/ceph/ceph.client.oceanbox.keyring"]; - #options = [ "name=csi-cephfs-provisioner" "secretfile=/etc/ceph/ceph.client.csi-cephfs-provisioner.keyring"]; - }; - }; - - environment.etc = { - "ceph/ceph.conf" = { - text = '' - [global] - mon_host=10.255.241.30:6789,10.255.241.31:6789,10.244.241.32:6789 - log file = /tmp/ceph-$pid.log - ''; - mode = "0644"; - }; - "ceph/ceph.client.oceanbox.keyring" = { - text = '' - AQDoLuhla6BWExAA6JTgxlsZkaVdxCj8GKM/UA== - ''; - mode = "0644"; - }; - "ceph/ceph.client.csi-cephfs-provisioner.keyring" = { - text = '' - AQAkmOFlL+WeDBAAZmt4Uwtv7duMyC0MNheXuw== - ''; - mode = "0644"; - }; - }; -} diff --git a/cluster/cluster.nix b/cluster/cluster.nix index b37fdc6..5b3e881 100644 --- a/cluster/cluster.nix +++ b/cluster/cluster.nix @@ -95,6 +95,7 @@ let }; cachix.enable = false; monitoring.nodeExporter.enable = false; + hpc.mft.enable = true; # Mellanox MFT }; networking = { @@ -189,7 +190,6 @@ let }; }; }; - inherit (import ./nfs.nix) fileSystems; }; k8s-node = { @@ -288,6 +288,7 @@ let environment.systemPackages = [ openssh-shosts pkgs.inotify-tools + pkgs.ceph pkgs.ceph-client ]; diff --git a/cluster/ekman/default.nix b/cluster/ekman/default.nix index 18f80ac..9ea3a88 100644 --- a/cluster/ekman/default.nix +++ b/cluster/ekman/default.nix @@ -19,6 +19,16 @@ in cluster = { compute = true; k8sNode = true; + mounts = { + rdma.enable = true; + automount.enable = false; + home = false; + opt = true; + work = true; + data = true; + backup = true; + ceph = false; + }; }; features = { @@ -292,9 +302,10 @@ in environment.systemPackages = []; imports = [ - ../cluster.nix - ../myvnc.nix ./hardware-configuration.nix + ../cluster.nix + ../mounts.nix + ../myvnc.nix ]; }; } diff --git a/cluster/fs0/default.nix b/cluster/fs0/default.nix index 1c768dc..dc00ab9 100644 --- a/cluster/fs0/default.nix +++ b/cluster/fs0/default.nix @@ -32,8 +32,18 @@ in { ]; cluster = { - k8sNode = true; - slurm = false; + k8sNode = true; + slurm = false; + mounts = { + rdma.enable = true; + automount.enable = true; + home = true; + opt = false; + work = true; + data = false; + backup = true; + ceph = false; + }; }; features.hpc.slurm.mungeUid = 994; @@ -150,16 +160,6 @@ in { device = "/vol/vol1"; options = [ "bind" ]; }; - "/work" = { - device = "10.255.243.90:/work"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" "vers=4.2" ]; - }; - # "/backup" = { - # device = "10.255.243.81:/backup"; - # fsType = "nfs"; - # options = [ "soft" "rdma" "defaults" "vers=4.2" ]; - # }; }; environment.etc = { @@ -207,10 +207,9 @@ in { programs.singularity.enable = true; imports = [ - ../cluster.nix ./hardware-configuration.nix + ../cluster.nix + ../mounts.nix ]; - - }; - + }; } diff --git a/cluster/fs1/default.nix b/cluster/fs1/default.nix index 9056028..92d9e0f 100644 --- a/cluster/fs1/default.nix +++ b/cluster/fs1/default.nix @@ -32,8 +32,18 @@ in { ]; cluster = { - k8sNode = true; - slurm = false; + k8sNode = true; + slurm = false; + mounts = { + rdma.enable = true; + automount.enable = true; + home = true; + opt = true; + work = false; + data = true; + backup = true; + ceph = false; + }; }; features.hpc.slurm.mungeUid = 994; @@ -135,67 +145,19 @@ in { services.rpcbind.enable = true; - systemd.mounts = [ - { - type = "nfs"; - what = "10.255.243.80:/data"; - where = "/data"; - mountConfig = { - Options = "soft,rdma,defaults,vers=4.2"; - }; - } - { - type = "nfs"; - what = "10.255.243.80:/opt"; - where = "/opt"; - mountConfig = { - Options = "soft,rdma,defaults,vers=4.2"; - }; - } - ]; - - systemd.automounts = [ - { - wantedBy = [ "multi-user.target" ]; - automountConfig = { - TimeoutIdleSec = "600"; - }; - where = "/data"; - } - { - wantedBy = [ "multi-user.target" ]; - automountConfig = { - TimeoutIdleSec = "600"; - }; - where = "/opt"; - } - ]; - fileSystems = { "/exports/work" = { device = "/work"; options = [ "bind" ]; }; - # "/exports/data" = { - # device = "/data"; - # options = [ "bind" ]; - # }; - # "/exports/opt" = { - # device = "/opt"; - # options = [ "bind" ]; - # }; - # "/vol/local-storage/vol1" = { - # device = "/vol/vol1"; - # options = [ "bind" ]; - # }; }; programs.singularity.enable = true; imports = [ - ../cluster.nix ./hardware-configuration.nix + ../cluster.nix + ../mounts.nix ]; - }; - + }; } diff --git a/cluster/fs2/default.nix b/cluster/fs2/default.nix index 888addc..544f443 100644 --- a/cluster/fs2/default.nix +++ b/cluster/fs2/default.nix @@ -54,8 +54,18 @@ in { }; cluster = { - k8sNode = true; - slurm = false; + k8sNode = true; + slurm = false; + mounts = { + rdma.enable = true; + automount.enable = true; + home = true; + opt = true; + work = true; + data = true; + backup = false; + ceph = false; + }; }; features.hpc.slurm.mungeUid = 996; @@ -159,50 +169,6 @@ in { services.rpcbind.enable = true; - systemd.mounts = [ - { - type = "nfs"; - what = "10.255.243.80:/data"; - where = "/data"; - mountConfig = { - Options = "soft,rdma,defaults,vers=4.2"; - }; - } - { - type = "nfs"; - what = "10.255.243.80:/opt"; - where = "/opt"; - mountConfig = { - Options = "soft,rdma,defaults,vers=4.2"; - }; - } - { - type = "nfs"; - what = "10.255.243.90:/work"; - where = "/work"; - mountConfig = { - Options = "soft,rdma,defaults,vers=4.2"; - }; - } - ]; - - systemd.automounts = [ - { - wantedBy = [ "multi-user.target" ]; - automountConfig = { - TimeoutIdleSec = "600"; - }; - where = "/data"; - } - { - wantedBy = [ "multi-user.target" ]; - automountConfig = { - TimeoutIdleSec = "600"; - }; - where = "/opt"; - } - ]; - fileSystems = { "/exports/backup" = { device = "/backup"; @@ -213,10 +179,10 @@ in { programs.singularity.enable = true; imports = [ - ../cluster.nix - ./hardware-configuration.nix - ../ceph.nix + ./hardware-configuration.nix + ../cluster.nix + ../mounts.nix ]; - }; + }; } diff --git a/cluster/mounts.nix b/cluster/mounts.nix new file mode 100644 index 0000000..5df807d --- /dev/null +++ b/cluster/mounts.nix @@ -0,0 +1,127 @@ +{ lib, config, ... }: +with lib; +let + cfg = config.cluster.mounts; + + subnet = if cfg.rdma.enable then "243" else "241"; + + options = + [ "soft" "defaults" "vers=4.2" ] ++ + (if cfg.rdma.enable then [ "rdma" ] else []) ++ + (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []); + + home = + if cfg.home then { + "/frontend" = { + device = "10.255.241.100:/home"; + fsType = "nfs4"; + options = [ + "soft" + "defaults" + "noauto" + "x-systemd.automount" + ]; + }; + } else {}; + + opt = + if cfg.opt then { + "/opt" = { + device = "10.255.${subnet}.80:/opt"; + fsType = "nfs4"; + inherit options; + }; + } else {}; + + data = + if cfg.data then { + "/data" = { + device = "10.255.${subnet}.80:/data"; + fsType = "nfs4"; + inherit options; + }; + } else {}; + + work = + if cfg.work then { + "/work" = { + device = "10.255.${subnet}.90:/work"; + fsType = "nfs4"; + inherit options; + }; + } else {}; + + backup = + if cfg.backup then { + "/backup" = { + device = "10.255.${subnet}.81:/backup"; + fsType = "nfs4"; + inherit options; + }; + } else {}; + + ceph = + if cfg.ceph then { + "/ceph" = { + device = "10.255.241.30/10.255.241.31/10.255.241.32:6789:/"; + fsType = "ceph"; + options = [ "name=oceanbox" "_netdev" "defaults" ]; + }; + } else {}; + + fileSystems = home // opt // data // work // backup // ceph; + + automount = mountpoint: + if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then + [{ + wantedBy = [ "multi-user.target" ]; + automountConfig = { + TimeoutIdleSec = "600"; + }; + where = mountpoint; + }] + else []; + + automounts = + [] ++ + automount "/work" ++ + automount "/opt" ++ + automount "/backup" ++ + automount "/data"; + + cephConf = + if cfg.ceph then { + "ceph/ceph.conf" = { + text = '' + [global] + mon_host = 10.255.241.30:6789,10.255.241.31:6789,10.244.241.32:6789 + log file = /tmp/ceph-$pid.log + [client.oceanbox] + key = AQDQNgRm6IE7JxAA1glJKsWPIBB/H/GxFYM0vQ== + [client.rbd] + key = AQCjth9mjR41ABAAvSs6hltidQT6Hu5OKwWu+Q== + ''; + mode = "0660"; + group = "admin"; + }; + } else {}; + +in +{ + options.cluster.mounts = { + rdma.enable = mkEnableOption "Enable NFS over RDMA"; + automount.enable = mkEnableOption "Enable NFS automounting"; + home = mkEnableOption "Enable /home"; + opt = mkEnableOption "Enable /opt"; + data = mkEnableOption "Enable /data"; + work = mkEnableOption "Enable /work"; + backup = mkEnableOption "Enable /backup"; + ceph = mkEnableOption "Enable /ceph"; + }; + + config = { + inherit fileSystems; + environment.etc = cephConf; + systemd.automounts = automounts; + }; +} diff --git a/cluster/nfs.nix b/cluster/nfs.nix deleted file mode 100644 index dc209a6..0000000 --- a/cluster/nfs.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - fileSystems = { - "/opt" = { - device = "10.255.243.80:/opt"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" "vers=4.2" ]; - }; - "/data" = { - device = "10.255.243.80:/data"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" "vers=4.2" ]; - }; - "/backup" = { - device = "10.255.243.81:/backup"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" "vers=4.2" ]; - }; - "/work" = { - device = "10.255.243.90:/work"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" "vers=4.2" ]; - }; - }; -} diff --git a/configuration.nix b/configuration.nix index b2ca2ee..a1d82ee 100644 --- a/configuration.nix +++ b/configuration.nix @@ -55,9 +55,19 @@ in { ]; cluster = { - k8sNode = true; - compute = false; - slurm = true; + k8sNode = true; + compute = false; + slurm = true; + mounts = { + rdma.enable = false; + automount.enable = true; + home = true; + opt = true; + work = true; + data = true; + backup = true; + ceph = false; + }; }; features = { @@ -168,6 +178,8 @@ in { # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" # ''; + services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0''; + networking = { useDHCP = false; hostName = name; @@ -199,15 +211,24 @@ in { } ]; }; - interfaces.ibp59s0 = { + interfaces.enp59s0np0 = { useDHCP = false; ipv4.addresses = [ { - address = ipoib; + address = "10.255.244.99"; prefixLength = 24; } ]; }; + # interfaces.ibp59s0 = { + # useDHCP = false; + # ipv4.addresses = [ + # { + # address = ipoib; + # prefixLength = 24; + # } + # ]; + # }; defaultGateway = "10.255.242.1"; firewall = { allowedTCPPorts = [ 4443 ]; @@ -289,9 +310,10 @@ in { ''; imports = [ - ./cluster/cluster.nix - ./cluster/myvnc.nix ./hardware-configuration.nix + ./cluster/cluster.nix + ./cluster/mounts.nix + ./cluster/myvnc.nix ]; } diff --git a/modules b/modules index 6c26659..156f5f2 160000 --- a/modules +++ b/modules @@ -1 +1 @@ -Subproject commit 6c26659a0b39c8a2a741e5c8d83421ef3f23aaed +Subproject commit 156f5f2452507bd5352c090d1c5bf1bb331fa7dd