fix: add k8s and hpc modules to main repo

2025-06-30 12:21:05 +02:00
parent 4aa9fa677a
commit bc3a034654
46 changed files with 4393 additions and 0 deletions
--- a/modules/hpc/hpc.nix
+++ b/modules/hpc/hpc.nix
@@ -0,0 +1,154 @@
+{ pkgs, lib, config, ... } :
+with lib;
+let
+  cfg = config.features.hpc;
+
+  configuration = {
+    programs.singularity.enable = true;
+
+    features.hpc.slurm.enable = mkDefault true;
+
+    environment.systemPackages = with pkgs; [
+      git
+      cmakeCurses
+      nco
+      neovim
+      python3
+      gfortran
+      # intel-mpi
+      # openmpi
+      rdma-core
+      mstflint
+      squashfsTools
+      linuxPackages.cpupower
+      linuxPackages.turbostat
+      hwloc
+    ];
+
+    services.openssh.extraConfig = ''
+      HostbasedAuthentication yes
+    '';
+
+    programs.ssh.extraConfig = ''
+      HostbasedAuthentication yes
+      EnableSSHKeysign yes
+    '';
+
+    powerManagement ={
+      enable = true;
+      cpuFreqGovernor = "performance";
+      powerUpCommands = ''
+        ${pkgs.linuxPackages.cpupower}/bin/cpupower -c 0-63 idle-set -d 2
+      '';
+    };
+
+    boot = {
+      # extraModulePackages = [ knem ];
+      # kernelModules = [ "knem" ];
+      kernel.sysctl = {
+        "kernel.randomize_va_space" = 0;
+      };
+    };
+
+    # services.udev.extraRules = ''
+    #   KERNEL=="knem", NAME="knem", GROUP="users", MODE="0660"
+    # '';
+
+    security.pam.services = {
+      sshd.limits = [ stackLimit memlockLimit ];
+      sudo.limits = [ stackLimit memlockLimit ];
+    };
+
+    programs.bash.shellInit = ''
+      ulimit -l unlimited
+      ulimit -s unlimited
+    '';
+
+    programs.fish.shellInit = ''
+      ulimit -l unlimited
+      ulimit -s unlimited
+    '';
+  };
+
+  frontend = {
+    services.influxdb.enable = true;
+    features.monitoring.nodeExporter.extraCollectors = [ "nfsd" ];
+  };
+
+  login = {
+    environment.systemPackages = with pkgs; [
+      # tigervnc
+      # tightvnc
+      turbovnc
+      emacs
+    ];
+    security.sudo.extraConfig = ''
+      %sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
+    '';
+  };
+
+  compute = {
+    boot.kernelParams = [ "mitigations=off" ]; # spectre/meltdown
+    features.monitoring.nodeExporter.extraCollectors = [ "nfs" ];
+  };
+
+  stackLimit = {
+    domain = "@users";
+    type = "hard";
+    item = "stack";
+    value = "unlimited";
+  };
+
+  memlockLimit = {
+    domain = "@users";
+    type = "hard";
+    item = "memlock";
+    value = "unlimited";
+  };
+
+  # intel-mpi = pkgs.callPackage ./intel-mpi.nix {};
+
+  knem =
+    let
+      kernel = config.boot.kernelPackages.kernel;
+      knem = pkgs.callPackage ./knem.nix { inherit kernel; };
+      # xpmem = pkgs.callPackage ./xpmem.nix { inherit kernel; };
+    in {
+      boot = {
+       kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
+       extraModulePackages = [ knem ];
+       kernelModules = [ "knem" ];
+      };
+
+    services.udev.extraRules = ''
+      KERNEL=="knem", NAME="knem", GROUP="users", MODE="0660"
+    '';
+  } ;
+
+in
+{
+  options.features.hpc = {
+    enable = mkEnableOption "Enable HPC features";
+
+    frontend = mkEnableOption "Enable frontend features";
+
+    login = mkEnableOption "Enable login node features";
+
+    compute = mkEnableOption "Enable compute features";
+
+    knem = mkEnableOption "Enable knem for openmpi";
+  };
+
+  config = mkIf cfg.enable (mkMerge [
+    configuration
+
+    (mkIf cfg.frontend frontend)
+
+    (mkIf cfg.login login)
+
+    (mkIf cfg.compute compute)
+
+    (mkIf cfg.knem knem)
+  ]);
+}
+