major: grand unified clusterfck (ekman not tested yet)

2025-09-12 13:12:36 +02:00
parent e0846164a7 ba5f1b8add
commit 5c72112457
97 changed files with 3704 additions and 8 deletions
--- a/ekman/fs-work/default.nix
+++ b/ekman/fs-work/default.nix
@@ -0,0 +1,171 @@
+{ pkgs ? import <nixpkgs> {} }:
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
+  #   sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
+  # }) {};
+  # pkgs = import <nixpkgs> {};
+  etcdCluster = import ../etcdCluster.nix;
+  name = "fs-work";
+  address = "10.255.241.90";
+in {
+  fs-work = { config, pkgs, ... }: with pkgs; {
+    deployment.tags = [ "fs" "fs-work" ];
+    deployment.targetHost = address;
+    system.autoUpgrade.enable = lib.mkForce false;
+
+    systemd.targets = {
+      sleep.enable = false;
+      suspend.enable = false;
+      hibernate.enable = false;
+      hybrid-sleep.enable = false;
+    };
+
+    # services.udev.extraRules = ''
+    #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+    # '';
+
+    environment.systemPackages = with pkgs; [
+        rdma-core
+        hwloc
+    ];
+
+    cluster = {
+      k8sNode = true;
+      slurm = false;
+      mounts = {
+        rdma.enable = false;
+        automount.enable = true;
+        users = false;
+        opt = false;
+        work = false;
+        data = false;
+        ceph = false;
+      };
+    };
+
+    features.hpc.slurm.mungeUid = 994;
+
+    features = {
+      host = {
+        inherit address;
+        inherit name;
+      };
+
+      os = {
+        externalInterface = "enp33s0f3np3";
+        nfs.enable = true;
+        nfs.exports = ''
+          /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          /exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+        '';
+      };
+
+      k8s = {
+          enable = true;
+          node.enable = true;
+          master.enable = false;
+          inherit etcdCluster;
+      };
+    };
+
+    systemd.services.rc-local = {
+        description = "rc.local script";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        path = [ "/run/current-system/sw/" ];
+        serviceConfig = {
+          Type = "oneshot";
+        };
+        script = ''
+            # if [ -e /sys/block/md126 ]; then
+            #     echo "deadline" > /sys/block/md126/queue/scheduler
+            #     # echo "4096" > /sys/block/md126/queue/nr_requests
+            #     echo "4096" > /sys/block/md126/queue/read_ahead_kb
+            #     echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
+            #     echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
+            # fi
+            grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
+            grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
+        '';
+    };
+
+    boot.kernel.sysctl = {
+        "vm.dirty_background_ratio" = 5;
+        "vm.dirty_ratio" = 10;
+        "vm.vfs_cache_pressure" = 50;
+        "vm.min_free_kbytes" = 262144;
+    };
+
+    networking = {
+      hostName = name;
+      interfaces.enp65s0f0np0 = {
+        useDHCP = false;
+        ipv4.addresses = [
+          {
+            address = address;
+            prefixLength = 24;
+          }
+        ];
+        ipv4.routes = [
+            {
+                address = "10.255.242.0";
+                prefixLength = 24;
+                via = "10.255.241.100";
+            }
+        ];
+      };
+      interfaces.enp1s0f1np1 = {
+        useDHCP = false;
+        ipv4.addresses = [
+          {
+            address = "10.255.244.90";
+            prefixLength = 24;
+          }
+        ];
+      };
+      # interfaces."ibp65s0.7666" = {
+      #   useDHCP = false;
+      # };
+      interfaces.ibp1s0f0 = {
+        useDHCP = false;
+        ipv4.addresses = [
+          {
+            address = "10.255.243.90";
+            prefixLength = 24;
+          }
+      ];
+      };
+      firewall = {
+        allowedTCPPorts = [];
+        allowedUDPPorts = [];
+        extraCommands = ''
+          # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
+        '';
+      };
+    };
+
+    services.rpcbind.enable = true;
+
+    fileSystems = {
+      "/exports/work" = {
+        device = "/work";
+        options = [ "bind" ];
+      };
+      "/exports/opt" = {
+        device = "/opt";
+        options = [ "bind" ];
+      };
+    };
+
+    programs.singularity.enable = true;
+
+    imports = [
+        ./hardware-configuration.nix
+        ../default.nix
+        ../mounts.nix
+    ];
+  };
+}