166 lines
3.6 KiB
Nix
166 lines
3.6 KiB
Nix
{ pkgs, lib, config, ... } :
|
|
with lib;
|
|
let
|
|
cfg = config.features.hpc;
|
|
|
|
configuration = {
|
|
programs.singularity.enable = true;
|
|
|
|
features.hpc.slurm.enable = mkDefault true;
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
git
|
|
cmakeCurses
|
|
nco
|
|
neovim
|
|
python3
|
|
gfortran
|
|
# intel-mpi
|
|
# openmpi
|
|
rdma-core
|
|
mstflint
|
|
squashfsTools
|
|
linuxPackages.cpupower
|
|
linuxPackages.turbostat
|
|
hwloc
|
|
];
|
|
|
|
services.openssh.extraConfig = ''
|
|
HostbasedAuthentication yes
|
|
'';
|
|
|
|
programs.ssh.extraConfig = ''
|
|
HostbasedAuthentication yes
|
|
EnableSSHKeysign yes
|
|
'';
|
|
|
|
powerManagement ={
|
|
enable = true;
|
|
cpuFreqGovernor = "performance";
|
|
powerUpCommands = ''
|
|
${pkgs.linuxPackages.cpupower}/bin/cpupower -c 0-63 idle-set -d 2
|
|
'';
|
|
};
|
|
|
|
boot = {
|
|
# extraModulePackages = [ knem ];
|
|
# kernelModules = [ "knem" ];
|
|
kernel.sysctl = {
|
|
"kernel.randomize_va_space" = 0;
|
|
};
|
|
};
|
|
|
|
# services.udev.extraRules = ''
|
|
# KERNEL=="knem", NAME="knem", GROUP="users", MODE="0660"
|
|
# '';
|
|
|
|
security.pam.services = {
|
|
sshd.limits = [ stackLimit memlockLimit ];
|
|
sudo.limits = [ stackLimit memlockLimit ];
|
|
};
|
|
|
|
programs.bash.shellInit = ''
|
|
ulimit -l unlimited
|
|
ulimit -s unlimited
|
|
'';
|
|
|
|
programs.fish.shellInit = ''
|
|
ulimit -l unlimited
|
|
ulimit -s unlimited
|
|
'';
|
|
};
|
|
|
|
frontend = {
|
|
services.influxdb.enable = true;
|
|
features.monitoring.nodeExporter.extraCollectors = [ "nfsd" ];
|
|
};
|
|
|
|
login = {
|
|
environment.systemPackages = with pkgs; [
|
|
# tigervnc
|
|
# tightvnc
|
|
turbovnc
|
|
emacs
|
|
];
|
|
security.sudo.extraConfig = ''
|
|
%sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
|
%admin ALL=(admin) NOPASSWD: ALL
|
|
'';
|
|
};
|
|
|
|
compute = {
|
|
boot.kernelParams = [ "mitigations=off" ]; # spectre/meltdown
|
|
features.monitoring.nodeExporter.extraCollectors = [ "nfs" ];
|
|
};
|
|
|
|
stackLimit = {
|
|
domain = "@users";
|
|
type = "hard";
|
|
item = "stack";
|
|
value = "unlimited";
|
|
};
|
|
|
|
memlockLimit = {
|
|
domain = "@users";
|
|
type = "hard";
|
|
item = "memlock";
|
|
value = "unlimited";
|
|
};
|
|
|
|
# intel-mpi = pkgs.callPackage ./intel-mpi.nix {};
|
|
|
|
knem =
|
|
let
|
|
kernel = config.boot.kernelPackages.kernel;
|
|
knem = pkgs.callPackage ./knem.nix { inherit kernel; };
|
|
# xpmem = pkgs.callPackage ./xpmem.nix { inherit kernel; };
|
|
in {
|
|
boot = {
|
|
#kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
|
|
kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override {
|
|
argsOverride = rec {
|
|
src = pkgs.fetchurl {
|
|
url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
|
|
sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid";
|
|
};
|
|
version = "5.10.239";
|
|
modDirVersion = "5.10.239";
|
|
};
|
|
});
|
|
extraModulePackages = [ knem ];
|
|
kernelModules = [ "knem" ];
|
|
};
|
|
|
|
services.udev.extraRules = ''
|
|
KERNEL=="knem", NAME="knem", GROUP="users", MODE="0660"
|
|
'';
|
|
} ;
|
|
|
|
in
|
|
{
|
|
options.features.hpc = {
|
|
enable = mkEnableOption "Enable HPC features";
|
|
|
|
frontend = mkEnableOption "Enable frontend features";
|
|
|
|
login = mkEnableOption "Enable login node features";
|
|
|
|
compute = mkEnableOption "Enable compute features";
|
|
|
|
knem = mkEnableOption "Enable knem for openmpi";
|
|
};
|
|
|
|
config = mkIf cfg.enable (mkMerge [
|
|
configuration
|
|
|
|
(mkIf cfg.frontend frontend)
|
|
|
|
(mkIf cfg.login login)
|
|
|
|
(mkIf cfg.compute compute)
|
|
|
|
(mkIf cfg.knem knem)
|
|
]);
|
|
}
|
|
|