fix: add k8s and hpc modules to main repo
This commit is contained in:
Jonas Juselius
96
modules/hpc/alert-rules.nix
Normal file
96
modules/hpc/alert-rules.nix
Normal file
@@ -0,0 +1,96 @@
|
||||
{ lib }:
|
||||
with lib;
|
||||
|
||||
let
|
||||
deviceFilter = ''fstype!="ramfs",device!="rpc_pipefs",device!="lxcfs",device!="nsfs",device!="borgfs"'';
|
||||
in mapAttrsToList (name: opts: {
|
||||
alert = name;
|
||||
expr = opts.condition;
|
||||
for = opts.time or "2m";
|
||||
labels = if (opts.page or true) then { severity = "page"; } else {};
|
||||
annotations = {
|
||||
summary = opts.summary;
|
||||
description = opts.description;
|
||||
};
|
||||
}) {
|
||||
watchdog = {
|
||||
condition = "vector(1)";
|
||||
summary = "An alert that should always be firing to certify that Alertmanager is working properly.";
|
||||
description = ''
|
||||
This is an alert meant to ensure that the entire alerting pipeline is functional.
|
||||
This alert is always firing, therefore it should always be firing in Alertmanager
|
||||
and always fire against a receiver. There are integrations with various notification
|
||||
mechanisms that send a notification when this alert is not firing. For example the
|
||||
"DeadMansSnitch" integration in PagerDuty.
|
||||
'';
|
||||
time = "12h";
|
||||
labels = { severity = "none"; };
|
||||
};
|
||||
node_down = {
|
||||
condition = ''up{job="node"} == 0'';
|
||||
summary = "{{$labels.alias}}: Node is down.";
|
||||
time = "10m";
|
||||
description = "{{$labels.alias}} has been down for more than 10 minutes.";
|
||||
};
|
||||
node_collector_failed = {
|
||||
condition = ''node_scrape_collector_success{job="node"} == 0'';
|
||||
summary = "{{$labels.alias}}: Node collector {{$labels.collector}} failed.";
|
||||
description = "{{$labels.alias}}: The collector {{$labels.collector}} of node exporter instance {{$labels.instance}} failed.";
|
||||
};
|
||||
node_systemd_service_failed = {
|
||||
condition = ''node_systemd_unit_state{state="failed"} == 1'';
|
||||
summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.";
|
||||
description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}.";
|
||||
};
|
||||
node_filesystem_full_90percent = {
|
||||
condition = ''sort(node_filesystem_free_bytes{${deviceFilter}} < node_filesystem_size_bytes{${deviceFilter}} * 0.1) / 1024^3'';
|
||||
time = "10m";
|
||||
page = false;
|
||||
summary = "{{$labels.alias}}: Filesystem is running out of space soon.";
|
||||
description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem.";
|
||||
};
|
||||
node_load15 = {
|
||||
condition = ''node_load15 / on(alias) count(node_cpu_seconds_total{mode="system"}) by (alias) >= 1.0 AND node_load15{alias !~ "c[0-9]-[0-9]"}'';
|
||||
time = "10m";
|
||||
page = false;
|
||||
summary = "{{$labels.alias}}: Running on high load: {{$value}}";
|
||||
description = "{{$labels.alias}} is running with load15 > 1 for at least 10 minutes: {{$value}}";
|
||||
};
|
||||
node_ram_using_90percent = {
|
||||
condition = ''node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1 AND node_memory_MemFree_bytes{alias !~ "c[0-9]-[0-9]"}'';
|
||||
time = "1h";
|
||||
page = false;
|
||||
summary = "{{$labels.alias}}: Using lots of RAM.";
|
||||
description = "{{$labels.alias}} is using at least 90% of its RAM for at least 1 hour.";
|
||||
};
|
||||
node_hwmon_temp = {
|
||||
condition = "node_hwmon_temp_crit_alarm_celsius == 1";
|
||||
summary = "{{$labels.alias}}: Sensor {{$labels.sensor}}/{{$labels.chip}} temp is high: {{$value}} ";
|
||||
description = "{{$labels.alias}} reports hwmon sensor {{$labels.sensor}}/{{$labels.chip}} temperature value is nearly critical: {{$value}}";
|
||||
};
|
||||
node_reboot = {
|
||||
condition = "time() - node_boot_time_seconds < 300";
|
||||
summary = "{{$labels.alias}}: Reboot";
|
||||
description = "{{$labels.alias}} just rebooted.";
|
||||
};
|
||||
node_uptime = {
|
||||
condition = "time() - node_boot_time_seconds > 2592000";
|
||||
page = false;
|
||||
summary = "{{$labels.alias}}: Uptime monster";
|
||||
description = "{{$labels.alias}} has been up for more than 30 days.";
|
||||
};
|
||||
slurm_nodes_offline = {
|
||||
condition = "slurm_node_down > 0 OR slurm_node_drain > 0 OR slurm_node_err > 0 OR slurm_node_fail > 0";
|
||||
summary = "Slurm nodes offline: {{$value}}";
|
||||
description = "Slurm node(s) have been offline for more than 5m.";
|
||||
};
|
||||
node_filesystem_full_in_7d = {
|
||||
condition = ''node_filesystem_free_bytes{${deviceFilter}} ''
|
||||
+ ''and predict_linear(node_filesystem_free_bytes{${deviceFilter}}[2d], 7*24*3600) <= 0'';
|
||||
page = false;
|
||||
time = "1h";
|
||||
summary = "{{$labels.alias}}: Filesystem is running out of space in 7 days.";
|
||||
description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space in approx. 7 days";
|
||||
};
|
||||
}
|
||||
|
||||
107
modules/hpc/beegfs/beegfs.nix
Normal file
107
modules/hpc/beegfs/beegfs.nix
Normal file
@@ -0,0 +1,107 @@
|
||||
{ pkgs, kernel ? pkgs.linux, ... } :
|
||||
with pkgs;
|
||||
let
|
||||
version = "7.4.0";
|
||||
in stdenvNoCC.mkDerivation {
|
||||
pname = "beegfs";
|
||||
inherit version;
|
||||
|
||||
src = fetchurl {
|
||||
name = "beegfs-archive-${version}.tar.bz2";
|
||||
# url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}";
|
||||
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2";
|
||||
sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ which unzip pkg-config cppunit perl makeWrapper ];
|
||||
|
||||
buildInputs = [
|
||||
gcc12
|
||||
libuuid
|
||||
attr
|
||||
xfsprogs
|
||||
zlib
|
||||
openssl
|
||||
sqlite
|
||||
rdma-core
|
||||
openssh
|
||||
gfortran
|
||||
influxdb
|
||||
curl
|
||||
rdma-core
|
||||
pahole
|
||||
];
|
||||
|
||||
hardeningDisable = [ "format" ]; # required for building beeond
|
||||
|
||||
postPatch = ''
|
||||
patchShebangs ./
|
||||
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
|
||||
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
|
||||
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
|
||||
'';
|
||||
|
||||
buildPhase = ''
|
||||
make BEEGFS_OPENTK_IBVERBS=1 \
|
||||
KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build \
|
||||
''${enableParallelBuilding:+-j''${NIX_BUILD_CORES} \
|
||||
-l''${NIX_BUILD_CORES}}
|
||||
'';
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
installPhase = ''
|
||||
binDir=$out/bin
|
||||
docDir=$out/share/doc/beegfs
|
||||
includeDir=$out/include/beegfs
|
||||
libDir=$out/lib
|
||||
libDirPkg=$out/lib/beegfs
|
||||
mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir
|
||||
cp common/build/libbeegfs_ib.so $libDir
|
||||
cp ctl/build/beegfs-ctl $binDir
|
||||
cp fsck/build/beegfs-fsck $binDir
|
||||
cp utils/scripts/beegfs-check-servers $binDir
|
||||
cp utils/scripts/beegfs-df $binDir
|
||||
cp utils/scripts/beegfs-net $binDir
|
||||
cp helperd/build/beegfs-helperd $binDir
|
||||
cp helperd/build/dist/etc/beegfs-helperd.conf $docDir
|
||||
cp client_module/build/dist/sbin/beegfs-setup-client $binDir
|
||||
cp client_module/build/dist/etc/beegfs-client.conf $docDir
|
||||
cp meta/build/beegfs-meta $binDir
|
||||
cp meta/build/dist/sbin/beegfs-setup-meta $binDir
|
||||
cp meta/build/dist/etc/beegfs-meta.conf $docDir
|
||||
cp mgmtd/build/beegfs-mgmtd $binDir
|
||||
cp mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir
|
||||
cp mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir
|
||||
cp storage/build/beegfs-storage $binDir
|
||||
cp storage/build/dist/sbin/beegfs-setup-storage $binDir
|
||||
cp storage/build/dist/etc/beegfs-storage.conf $docDir
|
||||
cp client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir
|
||||
cp -r client_devel/include/* $includeDir
|
||||
'';
|
||||
|
||||
# postFixup = ''
|
||||
# for i in $(find $out/bin -type f -executable); do
|
||||
# wrapProgram "$i" --prefix LD_LIBRARY_PATH : $out/lib
|
||||
# done
|
||||
# '';
|
||||
|
||||
doCheck = true;
|
||||
|
||||
# checkPhase = ''
|
||||
# LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/opentk_lib/build/ \
|
||||
# common/build/test-runner --text
|
||||
# '';
|
||||
|
||||
meta = with lib; {
|
||||
description = "High performance distributed filesystem with RDMA support";
|
||||
homepage = "https://www.beegfs.io";
|
||||
platforms = [ "i686-linux" "x86_64-linux" ];
|
||||
license = {
|
||||
fullName = "BeeGFS_EULA";
|
||||
url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
|
||||
free = true;
|
||||
};
|
||||
maintainers = with maintainers; [ "juselius" ];
|
||||
};
|
||||
}
|
||||
340
modules/hpc/beegfs/default.nix
Normal file
340
modules/hpc/beegfs/default.nix
Normal file
@@ -0,0 +1,340 @@
|
||||
{ config, lib, pkgs, ...} :
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.features.hpc.beegfs.beegfs;
|
||||
|
||||
# kernel = pkgs.linuxPackages_5_4.kernel;
|
||||
kernel = config.boot.kernelPackages.kernel;
|
||||
|
||||
beegfs = pkgs.callPackage ./beegfs.nix {
|
||||
inherit kernel;
|
||||
};
|
||||
beegfs-module = pkgs.callPackage ./kernel-module.nix {
|
||||
inherit kernel;
|
||||
};
|
||||
|
||||
# functions for the generations of config files
|
||||
|
||||
configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" ''
|
||||
storeMgmtdDirectory = ${cfg.mgmtd.storeDir}
|
||||
storeAllowFirstRunInit = false
|
||||
connAuthFile = ${cfg.connAuthFile}
|
||||
connPortShift = ${toString cfg.connPortShift}
|
||||
${cfg.mgmtd.extraConfig}
|
||||
'';
|
||||
|
||||
configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" ''
|
||||
storeMetaDirectory = ${cfg.meta.storeDir}
|
||||
sysMgmtdHost = ${cfg.mgmtdHost}
|
||||
connAuthFile = ${cfg.connAuthFile}
|
||||
connPortShift = ${toString cfg.connPortShift}
|
||||
storeAllowFirstRunInit = false
|
||||
${cfg.meta.extraConfig}
|
||||
'';
|
||||
|
||||
configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" ''
|
||||
storeStorageDirectory = ${cfg.storage.storeDir}
|
||||
sysMgmtdHost = ${cfg.mgmtdHost}
|
||||
connAuthFile = ${cfg.connAuthFile}
|
||||
connPortShift = ${toString cfg.connPortShift}
|
||||
storeAllowFirstRunInit = false
|
||||
${cfg.storage.extraConfig}
|
||||
'';
|
||||
|
||||
configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" ''
|
||||
connAuthFile = ${cfg.connAuthFile}
|
||||
${cfg.helperd.extraConfig}
|
||||
'';
|
||||
|
||||
configClientFilename = name : "/etc/beegfs/client-${name}.conf";
|
||||
|
||||
configClient = name: cfg: ''
|
||||
sysMgmtdHost = ${cfg.mgmtdHost}
|
||||
connAuthFile = ${cfg.connAuthFile}
|
||||
connPortShift = ${toString cfg.connPortShift}
|
||||
${cfg.client.extraConfig}
|
||||
'';
|
||||
|
||||
serviceList = [
|
||||
{ service = "meta"; cfgFile = configMeta; }
|
||||
{ service = "mgmtd"; cfgFile = configMgmtd; }
|
||||
{ service = "storage"; cfgFile = configStorage; }
|
||||
];
|
||||
|
||||
# functions to generate systemd.service entries
|
||||
|
||||
systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg:
|
||||
(nameValuePair "beegfs-${service}-${name}" (mkIf cfg.${service}.enable {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
requires = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
serviceConfig = rec {
|
||||
ExecStart = ''
|
||||
${beegfs}/bin/beegfs-${service} \
|
||||
cfgFile=${cfgFile name cfg} \
|
||||
pidFile=${PIDFile}
|
||||
'';
|
||||
PIDFile = "/run/beegfs-${service}-${name}.pid";
|
||||
TimeoutStopSec = "300";
|
||||
};
|
||||
}))) cfg);
|
||||
|
||||
systemdHelperd = mapAttrs' ( name: cfg:
|
||||
(nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
requires = [ "network-online.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
environment = {
|
||||
LD_LIBRARY_PATH = "${beegfs}/lib";
|
||||
};
|
||||
serviceConfig = rec {
|
||||
ExecStart = ''
|
||||
${beegfs}/bin/beegfs-helperd \
|
||||
cfgFile=${configHelperd name cfg} \
|
||||
pidFile=${PIDFile}
|
||||
'';
|
||||
PIDFile = "/run/beegfs-helperd-${name}.pid";
|
||||
TimeoutStopSec = "300";
|
||||
};
|
||||
}))) cfg;
|
||||
|
||||
# wrappers to beegfs tools. Avoid typing path of config files
|
||||
utilWrappers = mapAttrsToList ( name: cfg:
|
||||
( pkgs.runCommand "beegfs-utils-${name}" {
|
||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||
preferLocalBuild = true;
|
||||
} ''
|
||||
mkdir -p $out/bin
|
||||
makeWrapper ${beegfs}/bin/beegfs-check-servers \
|
||||
$out/bin/beegfs-check-servers-${name} \
|
||||
--add-flags "-c ${configClientFilename name}" \
|
||||
--prefix PATH : ${lib.makeBinPath [ beegfs ]}
|
||||
makeWrapper ${beegfs}/bin/beegfs-ctl \
|
||||
$out/bin/beegfs-ctl-${name} \
|
||||
--add-flags "--cfgFile=${configClientFilename name}"
|
||||
makeWrapper ${beegfs}/bin/beegfs-ctl \
|
||||
$out/bin/beegfs-df-${name} \
|
||||
--add-flags "--cfgFile=${configClientFilename name}" \
|
||||
--add-flags --listtargets \
|
||||
--add-flags --hidenodeid \
|
||||
--add-flags --pools \
|
||||
--add-flags --spaceinfo
|
||||
makeWrapper ${beegfs}/bin/beegfs-fsck \
|
||||
$out/bin/beegfs-fsck-${name} \
|
||||
--add-flags "--cfgFile=${configClientFilename name}"
|
||||
''
|
||||
)) cfg;
|
||||
beegfsOptions = {
|
||||
options = {
|
||||
mgmtdHost = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
example = "master";
|
||||
description = ''Hostname of managament host.'';
|
||||
};
|
||||
|
||||
connAuthFile = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
example = "/etc/my.key";
|
||||
description = "File containing shared secret authentication.";
|
||||
};
|
||||
|
||||
connPortShift = mkOption {
|
||||
type = types.int;
|
||||
default = 0;
|
||||
example = 5;
|
||||
description = ''
|
||||
For each additional beegfs configuration shift all
|
||||
service TCP/UDP ports by at least 5.
|
||||
'';
|
||||
};
|
||||
|
||||
client = {
|
||||
enable = mkEnableOption "BeeGFS client";
|
||||
|
||||
mount = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Create fstab entry automatically";
|
||||
};
|
||||
|
||||
mountPoint = mkOption {
|
||||
type = types.str;
|
||||
default = "/run/beegfs";
|
||||
description = ''
|
||||
Mount point under which the beegfs filesytem should be mounted.
|
||||
If mounted manually the mount option specifing the config file is needed:
|
||||
cfgFile=/etc/beegfs/beegfs-client-<name>.conf
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = types.lines;
|
||||
default = "";
|
||||
description = ''
|
||||
Additional lines for beegfs-client.conf.
|
||||
See documentation for further details.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
helperd = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = ''
|
||||
Enable the BeeGFS helperd.
|
||||
The helpered is need for logging purposes on the client.
|
||||
Disabling <literal>helperd</literal> allows for runing the client
|
||||
with <literal>allowUnfree = false</literal>.
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = types.lines;
|
||||
default = "";
|
||||
description = ''
|
||||
Additional lines for beegfs-helperd.conf. See documentation
|
||||
for further details.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
mgmtd = {
|
||||
enable = mkEnableOption "BeeGFS mgmtd daemon";
|
||||
|
||||
storeDir = mkOption {
|
||||
type = types.path;
|
||||
default = null;
|
||||
example = "/data/beegfs-mgmtd";
|
||||
description = ''
|
||||
Data directory for mgmtd.
|
||||
Must not be shared with other beegfs daemons.
|
||||
This directory must exist and it must be initialized
|
||||
with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p <storeDir>"
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = types.lines;
|
||||
default = "";
|
||||
description = ''
|
||||
Additional lines for beegfs-mgmtd.conf. See documentation
|
||||
for further details.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
meta = {
|
||||
enable = mkEnableOption "BeeGFS meta data daemon";
|
||||
|
||||
storeDir = mkOption {
|
||||
type = types.path;
|
||||
default = null;
|
||||
example = "/data/beegfs-meta";
|
||||
description = ''
|
||||
Data directory for meta data service.
|
||||
Must not be shared with other beegfs daemons.
|
||||
The underlying filesystem must be mounted with xattr turned on.
|
||||
This directory must exist and it must be initialized
|
||||
with beegfs-setup-meta, e.g.
|
||||
"beegfs-setup-meta -C -s <serviceID> -p <storeDir>"
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
description = ''
|
||||
Additional lines for beegfs-meta.conf. See documentation
|
||||
for further details.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
storage = {
|
||||
enable = mkEnableOption "BeeGFS storage daemon";
|
||||
|
||||
storeDir = mkOption {
|
||||
type = types.path;
|
||||
default = null;
|
||||
example = "/data/beegfs-storage";
|
||||
description = ''
|
||||
Data directories for storage service.
|
||||
Must not be shared with other beegfs daemons.
|
||||
The underlying filesystem must be mounted with xattr turned on.
|
||||
This directory must exist and it must be initialized
|
||||
with beegfs-setup-storage, e.g.
|
||||
"beegfs-setup-storage -C -s <serviceID> -i <storageTargetID> -p <storeDir>"
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
description = ''
|
||||
Addional lines for beegfs-storage.conf. See documentation
|
||||
for further details.
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
###### interface
|
||||
|
||||
options.features.hpc.beegfs = {
|
||||
enable = mkEnableOption "BeeGFS";
|
||||
|
||||
beegfs = mkOption {
|
||||
type = with types; attrsOf (submodule ({ ... } : beegfsOptions ));
|
||||
default = {};
|
||||
description = ''
|
||||
BeeGFS configurations. Every mount point requires a separate configuration.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
###### implementation
|
||||
|
||||
config = mkIf config.features.hpc.beegfs.enable {
|
||||
|
||||
environment.systemPackages = utilWrappers;
|
||||
|
||||
# Put the client.conf files in /etc since they are needed
|
||||
# by the commandline tools
|
||||
environment.etc = mapAttrs' ( name: cfg:
|
||||
(nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable)
|
||||
{
|
||||
enable = true;
|
||||
text = configClient name cfg;
|
||||
}))) cfg;
|
||||
|
||||
# Kernel module, we need it only once per host.
|
||||
boot = mkIf (
|
||||
foldr (a: b: a || b) false
|
||||
(map (x: x.client.enable) (collect (x: x ? client) cfg)))
|
||||
{
|
||||
kernelModules = [ "beegfs" ];
|
||||
extraModulePackages = [ beegfs-module ];
|
||||
};
|
||||
|
||||
# generate fstab entries
|
||||
fileSystems = mapAttrs' (name: cfg:
|
||||
(nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable {
|
||||
device = "beegfs_nodev";
|
||||
fsType = "beegfs";
|
||||
mountPoint = cfg.client.mountPoint;
|
||||
options = [ "cfgFile=${configClientFilename name}" "_netdev" ];
|
||||
})))) cfg;
|
||||
|
||||
# generate systemd services
|
||||
systemd.services = systemdHelperd //
|
||||
foldr (a: b: a // b) {}
|
||||
(map (x: systemdEntry x.service x.cfgFile) serviceList);
|
||||
};
|
||||
}
|
||||
50
modules/hpc/beegfs/kernel-module.nix
Normal file
50
modules/hpc/beegfs/kernel-module.nix
Normal file
@@ -0,0 +1,50 @@
|
||||
{ pkgs, kernel ? pkgs.linux, ... } :
|
||||
with pkgs;
|
||||
let
|
||||
version = "7.4.0";
|
||||
beegfs = pkgs.callPackage ./beegfs.nix { inherit kernel; };
|
||||
in stdenvNoCC.mkDerivation {
|
||||
name = "beegfs-module-${version}-${kernel.version}";
|
||||
|
||||
src = fetchurl {
|
||||
name = "beegfs-archive-${version}.tar.bz2";
|
||||
# url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}";
|
||||
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2";
|
||||
sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo=";
|
||||
};
|
||||
|
||||
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
|
||||
|
||||
nativeBuildInputs = [ gcc12 which kmod pahole ];
|
||||
|
||||
buildInputs = kernel.moduleBuildDependencies;
|
||||
|
||||
makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ];
|
||||
|
||||
postPatch = ''
|
||||
patchShebangs ./
|
||||
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
|
||||
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
|
||||
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
|
||||
find -type f -name "configure" -exec sed -i "s:/bin/:/usr/bin/env :" \{} \;
|
||||
find -type f -name "configure" -exec sed -i "s:/usr/bin/:/usr/bin/env :" \{} \;
|
||||
sed -i 's,libbeegfs_ib.so,${beegfs}/lib/&,' common/source/common/net/sock/RDMASocket.cpp
|
||||
'';
|
||||
|
||||
preBuild = "cd client_module/build";
|
||||
|
||||
installPhase = ''
|
||||
instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
|
||||
mkdir -p $instdir
|
||||
cp beegfs.ko $instdir
|
||||
'';
|
||||
|
||||
meta = with lib; {
|
||||
description = "High performance distributed filesystem with RDMA support";
|
||||
homepage = "https://www.beegfs.io";
|
||||
platforms = [ "i686-linux" "x86_64-linux" ];
|
||||
license = licenses.gpl2;
|
||||
maintainers = with maintainers; [ "juselius" ];
|
||||
# broken = stdenv.lib.versionAtLeast kernel.version "4.18";
|
||||
};
|
||||
}
|
||||
9
modules/hpc/default.nix
Normal file
9
modules/hpc/default.nix
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
imports = [
|
||||
./beegfs
|
||||
./hpc.nix
|
||||
./slurm.nix
|
||||
./monitoring.nix
|
||||
./mft
|
||||
];
|
||||
}
|
||||
154
modules/hpc/hpc.nix
Normal file
154
modules/hpc/hpc.nix
Normal file
@@ -0,0 +1,154 @@
|
||||
{ pkgs, lib, config, ... } :
|
||||
with lib;
|
||||
let
|
||||
cfg = config.features.hpc;
|
||||
|
||||
configuration = {
|
||||
programs.singularity.enable = true;
|
||||
|
||||
features.hpc.slurm.enable = mkDefault true;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
git
|
||||
cmakeCurses
|
||||
nco
|
||||
neovim
|
||||
python3
|
||||
gfortran
|
||||
# intel-mpi
|
||||
# openmpi
|
||||
rdma-core
|
||||
mstflint
|
||||
squashfsTools
|
||||
linuxPackages.cpupower
|
||||
linuxPackages.turbostat
|
||||
hwloc
|
||||
];
|
||||
|
||||
services.openssh.extraConfig = ''
|
||||
HostbasedAuthentication yes
|
||||
'';
|
||||
|
||||
programs.ssh.extraConfig = ''
|
||||
HostbasedAuthentication yes
|
||||
EnableSSHKeysign yes
|
||||
'';
|
||||
|
||||
powerManagement ={
|
||||
enable = true;
|
||||
cpuFreqGovernor = "performance";
|
||||
powerUpCommands = ''
|
||||
${pkgs.linuxPackages.cpupower}/bin/cpupower -c 0-63 idle-set -d 2
|
||||
'';
|
||||
};
|
||||
|
||||
boot = {
|
||||
# extraModulePackages = [ knem ];
|
||||
# kernelModules = [ "knem" ];
|
||||
kernel.sysctl = {
|
||||
"kernel.randomize_va_space" = 0;
|
||||
};
|
||||
};
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="knem", NAME="knem", GROUP="users", MODE="0660"
|
||||
# '';
|
||||
|
||||
security.pam.services = {
|
||||
sshd.limits = [ stackLimit memlockLimit ];
|
||||
sudo.limits = [ stackLimit memlockLimit ];
|
||||
};
|
||||
|
||||
programs.bash.shellInit = ''
|
||||
ulimit -l unlimited
|
||||
ulimit -s unlimited
|
||||
'';
|
||||
|
||||
programs.fish.shellInit = ''
|
||||
ulimit -l unlimited
|
||||
ulimit -s unlimited
|
||||
'';
|
||||
};
|
||||
|
||||
frontend = {
|
||||
services.influxdb.enable = true;
|
||||
features.monitoring.nodeExporter.extraCollectors = [ "nfsd" ];
|
||||
};
|
||||
|
||||
login = {
|
||||
environment.systemPackages = with pkgs; [
|
||||
# tigervnc
|
||||
# tightvnc
|
||||
turbovnc
|
||||
emacs
|
||||
];
|
||||
security.sudo.extraConfig = ''
|
||||
%sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
||||
'';
|
||||
};
|
||||
|
||||
compute = {
|
||||
boot.kernelParams = [ "mitigations=off" ]; # spectre/meltdown
|
||||
features.monitoring.nodeExporter.extraCollectors = [ "nfs" ];
|
||||
};
|
||||
|
||||
stackLimit = {
|
||||
domain = "@users";
|
||||
type = "hard";
|
||||
item = "stack";
|
||||
value = "unlimited";
|
||||
};
|
||||
|
||||
memlockLimit = {
|
||||
domain = "@users";
|
||||
type = "hard";
|
||||
item = "memlock";
|
||||
value = "unlimited";
|
||||
};
|
||||
|
||||
# intel-mpi = pkgs.callPackage ./intel-mpi.nix {};
|
||||
|
||||
knem =
|
||||
let
|
||||
kernel = config.boot.kernelPackages.kernel;
|
||||
knem = pkgs.callPackage ./knem.nix { inherit kernel; };
|
||||
# xpmem = pkgs.callPackage ./xpmem.nix { inherit kernel; };
|
||||
in {
|
||||
boot = {
|
||||
kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
|
||||
extraModulePackages = [ knem ];
|
||||
kernelModules = [ "knem" ];
|
||||
};
|
||||
|
||||
services.udev.extraRules = ''
|
||||
KERNEL=="knem", NAME="knem", GROUP="users", MODE="0660"
|
||||
'';
|
||||
} ;
|
||||
|
||||
in
|
||||
{
|
||||
options.features.hpc = {
|
||||
enable = mkEnableOption "Enable HPC features";
|
||||
|
||||
frontend = mkEnableOption "Enable frontend features";
|
||||
|
||||
login = mkEnableOption "Enable login node features";
|
||||
|
||||
compute = mkEnableOption "Enable compute features";
|
||||
|
||||
knem = mkEnableOption "Enable knem for openmpi";
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable (mkMerge [
|
||||
configuration
|
||||
|
||||
(mkIf cfg.frontend frontend)
|
||||
|
||||
(mkIf cfg.login login)
|
||||
|
||||
(mkIf cfg.compute compute)
|
||||
|
||||
(mkIf cfg.knem knem)
|
||||
]);
|
||||
}
|
||||
|
||||
58
modules/hpc/infiniband-exporter.nix
Normal file
58
modules/hpc/infiniband-exporter.nix
Normal file
@@ -0,0 +1,58 @@
|
||||
{pkgs, config, lib, ...}:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.features.monitoring.infiniband-exporter;
|
||||
|
||||
python-env = pkgs.python3.withPackages (ps: with ps; [
|
||||
prometheus_client
|
||||
]
|
||||
);
|
||||
|
||||
exporter = pkgs.fetchFromGitHub {
|
||||
owner = "guilbaults";
|
||||
repo = "infiniband-exporter";
|
||||
rev = "12e7b2de049fc3c33c44e164f426dd723c8479c0";
|
||||
hash = "sha256-+n09beiJEgOgX+3po7fjiwZrziug+5N4JHi7ivTYa9U=";
|
||||
};
|
||||
|
||||
nameMap = pkgs.writeTextFile {
|
||||
name = "infiniband-node-name-map.txt";
|
||||
text = cfg.nameMap;
|
||||
};
|
||||
|
||||
infiniband-exporter-service = {
|
||||
systemd.services.infiniband-exporter = {
|
||||
enable = true;
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
description = "Prometheus InfiniBand exporter";
|
||||
path = [ pkgs.rdma-core ];
|
||||
script = "${python-env}/bin/python3 ${exporter}/infiniband-exporter.py"
|
||||
+ " --port ${builtins.toString cfg.port} --can-reset-counter"
|
||||
+ (if cfg.nameMap == null then "" else " --node-name-map=${nameMap}");
|
||||
serviceConfig = {
|
||||
RestartSec = "15s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
options.features.monitoring.infiniband-exporter = {
|
||||
enable = mkEnableOption "Enable InfiniBand prometheus exporter";
|
||||
|
||||
port = mkOption {
|
||||
type = types.ints.unsigned;
|
||||
default = 9683;
|
||||
description = "Collector http port";
|
||||
};
|
||||
|
||||
nameMap = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Node name map";
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable infiniband-exporter-service;
|
||||
}
|
||||
69
modules/hpc/intel-mpi.nix
Normal file
69
modules/hpc/intel-mpi.nix
Normal file
@@ -0,0 +1,69 @@
|
||||
{ stdenv, lib, glibc, gcc, file , patchelf , makeWrapper }:
|
||||
|
||||
let
|
||||
|
||||
preinstDir = "opt/intel/oneapi/mpi/${version}";
|
||||
version = "2021.1.1";
|
||||
|
||||
self = stdenv.mkDerivation rec {
|
||||
inherit version;
|
||||
name = "intelmpi-${version}";
|
||||
src = ./intel-mpi.tgz;
|
||||
|
||||
nativeBuildInputs= [ file patchelf makeWrapper ];
|
||||
|
||||
dontPatchELF = true;
|
||||
dontStrip = true;
|
||||
|
||||
installPhase = ''
|
||||
mpi=$out/opt/intel/oneapi/mpi
|
||||
mkdir -p $mpi
|
||||
cp -r * $mpi
|
||||
cp -rs $mpi/${version}/bin $out
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
echo "Patching rpath and interpreter..."
|
||||
for f in $(find $out -type f -executable); do
|
||||
type="$(file -b --mime-type $f)"
|
||||
case "$type" in
|
||||
"application/executable"|"application/x-executable")
|
||||
echo "Patching executable: $f"
|
||||
patchelf --set-interpreter $(echo ${glibc}/lib/ld-linux*.so.2) --set-rpath ${glibc}/lib:\$ORIGIN:\$ORIGIN/../lib $f || true
|
||||
;;
|
||||
"application/x-sharedlib"|"application/x-pie-executable")
|
||||
echo "Patching library: $f"
|
||||
patchelf --set-rpath ${glibc}/lib:\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../../libfabric/lib $f || true
|
||||
;;
|
||||
*)
|
||||
echo "$f ($type) not patched"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "Fixing path into scripts..."
|
||||
for file in `grep -l -r "/${preinstDir}" $out`; do
|
||||
sed -e "s,/${preinstDir},$out,g" -i $file
|
||||
done
|
||||
for file in `grep -l -r "I_MPI_SUBSTITUTE_INSTALLDIR" $out`; do
|
||||
sed -e "s,I_MPI_SUBSTITUTE_INSTALLDIR,$out,g" -i $file
|
||||
done
|
||||
|
||||
wrapProgram $out/${preinstDir}/bin/mpiexec.hydra \
|
||||
--set UCX_TLS ud,sm,self \
|
||||
--set I_MPI_FABRICS shm:ofi \
|
||||
--set FI_PROVIDER_PATH $out/${preinstDir}/libfabric/lib/prov \
|
||||
--set FI_PROVIDER mlx
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
isIntel = true;
|
||||
};
|
||||
|
||||
meta = {
|
||||
description = "Intel MPI ${version} library";
|
||||
maintainers = [ lib.maintainers.dguibert ];
|
||||
platforms = lib.platforms.linux;
|
||||
};
|
||||
};
|
||||
in self
|
||||
|
||||
46
modules/hpc/knem.nix
Normal file
46
modules/hpc/knem.nix
Normal file
@@ -0,0 +1,46 @@
|
||||
{ pkgs, kernel ? pkgs.linux, ... } :
|
||||
with pkgs;
|
||||
let
|
||||
version = "master";
|
||||
kdir="${kernel.dev}/lib/modules/${kernel.modDirVersion}";
|
||||
in stdenv.mkDerivation {
|
||||
inherit version;
|
||||
name = "knem-${version}-${kernel.version}";
|
||||
|
||||
# src = fetchurl {
|
||||
# name = "knem-${version}.tar.bz2";
|
||||
# url = "https://gitlab.inria.fr/knem/knem/uploads/4a43e3eb860cda2bbd5bf5c7c04a24b6/knem-1.1.4.tar.gz";
|
||||
# sha256 = "0dq9a41s08alrgggabmlyagmwl95sczmhi36gph5axmfg42kc3lz";
|
||||
# };
|
||||
|
||||
src = fetchgit {
|
||||
name = "knem-${version}";
|
||||
url = "https://gitlab.inria.fr/knem/knem.git";
|
||||
sha256 = "sha256-ptjALI2q2AF0tvdxXm4xH+8rXO8qnRwPfWMPITjrKVI=";
|
||||
};
|
||||
|
||||
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
|
||||
|
||||
nativeBuildInputs = [ which kmod pkgconf ];
|
||||
|
||||
buildInputs = kernel.moduleBuildDependencies ++ [
|
||||
libtool autoconf pkgconf automake hwloc
|
||||
] ;
|
||||
|
||||
preConfigurePhases = "preConfigure";
|
||||
preConfigure = ''
|
||||
autoupdate
|
||||
./autogen.sh
|
||||
'';
|
||||
|
||||
configureFlags = [
|
||||
"--with-linux-release=${kernel.modDirVersion}"
|
||||
"--with-linux=${kdir}/source"
|
||||
"--with-linux-build=${kdir}/build"
|
||||
];
|
||||
|
||||
installPhase = ''
|
||||
make install
|
||||
rm -rf $out/etc $out/sbin
|
||||
'';
|
||||
}
|
||||
29
modules/hpc/mft/default.nix
Normal file
29
modules/hpc/mft/default.nix
Normal file
@@ -0,0 +1,29 @@
|
||||
{config, lib, pkgs,...}:
|
||||
with lib;
|
||||
let
|
||||
kernel = config.boot.kernelPackages.kernel;
|
||||
|
||||
mft = pkgs.callPackage ./mft.nix { inherit kernel; };
|
||||
in
|
||||
{
|
||||
###### interface
|
||||
|
||||
options.features.hpc.mft = {
|
||||
enable = mkEnableOption "Mellanox MFT";
|
||||
};
|
||||
|
||||
###### implementation
|
||||
|
||||
config = mkIf config.features.hpc.mft.enable {
|
||||
environment.etc."mft/mft.conf".source = "${mft.mft}/etc/mft/mft.conf";
|
||||
environment.etc."mft/mst.conf".source = "${mft.mft}/etc/mft/mst.conf";
|
||||
environment.etc."mft/ca-bundle.crt".source = "${mft.mft}/etc/mft/ca-bundle.crt";
|
||||
|
||||
environment.systemPackages = [ pkgs.pciutils mft.mft ];
|
||||
|
||||
# boot = {
|
||||
# kernelModules = [ "mst_pci" "mst_pciconf" ];
|
||||
# extraModulePackages = [ mft.mft-kernel-module ];
|
||||
# };
|
||||
};
|
||||
}
|
||||
123
modules/hpc/mft/mft.nix
Normal file
123
modules/hpc/mft/mft.nix
Normal file
@@ -0,0 +1,123 @@
|
||||
{pkgs, lib, stdenv, kernel ? pkgs.linux, ...}:
|
||||
let
|
||||
version = "4.27.0";
|
||||
ver = "${version}-83";
|
||||
arch = "amd64";
|
||||
|
||||
rpath = lib.strings.concatStringsSep ":" [
|
||||
"${pkgs.libxcrypt}/lib"
|
||||
"${pkgs.glibc}/lib"
|
||||
"${stdenv.cc.cc.lib.outPath}/lib"
|
||||
];
|
||||
|
||||
src = pkgs.fetchurl {
|
||||
url = "https://www.mellanox.com/downloads/MFT/mft-${ver}-x86_64-deb.tgz";
|
||||
hash = "sha256-Mx2dyHSFkZ+vsorAd7yVe2vU8nhksoGieE+LPcA5fZA=";
|
||||
};
|
||||
|
||||
unpackPhase = ''
|
||||
PATH=${pkgs.dpkg}/bin:$PATH
|
||||
tar vfxz $src
|
||||
mv mft-${ver}-x86_64-deb deb
|
||||
'';
|
||||
|
||||
preFixup = ''
|
||||
for i in $out/usr/bin/*; do
|
||||
if $(file $i | grep -q 'ELF.*dynamic'); then
|
||||
patchelf \
|
||||
--set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
|
||||
--set-rpath "${rpath}" $i
|
||||
elif $(file $i | grep -q shell); then
|
||||
patchShebangs --build $i
|
||||
fi
|
||||
done
|
||||
'';
|
||||
|
||||
in
|
||||
rec {
|
||||
mft = stdenv.mkDerivation {
|
||||
name = "mft-${ver}";
|
||||
inherit src unpackPhase preFixup;
|
||||
|
||||
installPhase = ''
|
||||
PATH=/bin:$PATH
|
||||
dpkg -x deb/DEBS/mft_${ver}_${arch}.deb $out
|
||||
rm $out/usr/bin/mst
|
||||
mv $out/etc/init.d/mst $out/usr/bin/mst
|
||||
rmdir $out/etc/init.d
|
||||
sed -i "15i export PATH=/run/current-system/sw/bin:${pkgs.kmod}/bin
|
||||
s,/usr/mst,$out&,;
|
||||
s,/sbin/modprobe,modprobe,;
|
||||
s,/sbin/lsmod,lsmod,;
|
||||
s,lsmod,${pkgs.kmod}/bin/lsmod,;
|
||||
s,modprobe \+-r,${pkgs.kmod}/bin/rmmod,;
|
||||
s,=lspci,=${pkgs.pciutils}/bin/lspci,;
|
||||
s,mbindir=,&$out,;
|
||||
s,mlibdir=,&$out,;
|
||||
s,MST_PCI_MOD=.*,MST_PCI_MOD="${mft-kernel-module}/lib/modules/${kernel.version}/extras/mft/mst_pci.ko,";
|
||||
s,MST_PCICONF_MOD=.*,MST_PCICONF_MOD="${mft-kernel-module}/lib/modules/${kernel.version}/extras/mft/mst_pciconf.ko,";
|
||||
s,^PATH=.*,PATH=\$\{PATH\}:\$\{mbindir\},;" $out/usr/bin/mst
|
||||
sed -i "s,mft_prefix_location=.*,mft_prefix_location=$out/usr," $out/etc/mft/mft.conf
|
||||
mkdir $out/bin
|
||||
cp -s $out/usr/bin/* $out/bin
|
||||
'';
|
||||
};
|
||||
|
||||
oem = stdenv.mkDerivation {
|
||||
name = "mft-oem-${ver}";
|
||||
inherit src unpackPhase preFixup;
|
||||
|
||||
installPhase = ''
|
||||
PATH=/bin:$PATH
|
||||
dpkg -x deb/DEBS/mft-oem_${ver}_${arch}.deb $out
|
||||
'';
|
||||
};
|
||||
|
||||
pcap = stdenv.mkDerivation {
|
||||
name = "mft-pcap${ver}";
|
||||
inherit src unpackPhase preFixup;
|
||||
|
||||
installPhase = ''
|
||||
PATH=/bin:$PATH
|
||||
dpkg -x deb/DEBS/mft-pcap_${ver}_${arch}.deb $out
|
||||
'';
|
||||
};
|
||||
|
||||
mft-kernel-module = stdenv.mkDerivation {
|
||||
name = "mft-kernel-module";
|
||||
pname = "mft-kernel-module";
|
||||
inherit src;
|
||||
inherit unpackPhase;
|
||||
|
||||
prePatch = ''
|
||||
PATH=/bin:$PATH
|
||||
dpkg -x deb/SDEBS/kernel-mft-dkms_${ver}_all.deb source
|
||||
'';
|
||||
|
||||
preConfigure = ''
|
||||
export KSRC="${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
|
||||
export sourceRoot="/build/source/usr/src/kernel-mft-dkms-${version}"
|
||||
buildRoot () { echo $KSRC; }
|
||||
'';
|
||||
|
||||
nativeBuildInputs = kernel.moduleBuildDependencies;
|
||||
|
||||
buildPhase = ''
|
||||
cd $sourceRoot/mst_backward_compatibility/mst_pci
|
||||
make ${lib.strings.concatStringsSep " " kernel.makeFlags} -C "${kernel.dev}/lib/modules/${kernel.modDirVersion}/build" M=$(pwd) modules
|
||||
cd $sourceRoot/mst_backward_compatibility/mst_pciconf
|
||||
make ${lib.strings.concatStringsSep " " kernel.makeFlags} -C "${kernel.dev}/lib/modules/${kernel.modDirVersion}/build" M=$(pwd) modules
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
instdir=$out/lib/modules/${kernel.modDirVersion}/extras/mft
|
||||
mkdir -p $instdir
|
||||
cp $sourceRoot/mst_backward_compatibility/mst_pci/mst_pci.ko $instdir
|
||||
cp $sourceRoot/mst_backward_compatibility/mst_pciconf/mst_pciconf.ko $instdir
|
||||
'';
|
||||
|
||||
meta = {
|
||||
description = "Mellanox MFT kernel module";
|
||||
};
|
||||
};
|
||||
}
|
||||
277
modules/hpc/monitoring.nix
Normal file
277
modules/hpc/monitoring.nix
Normal file
@@ -0,0 +1,277 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.features.monitoring;
|
||||
|
||||
mkScrapeConfigs = configs: flip mapAttrsToList configs (k: v:
|
||||
let
|
||||
static_configs = flip map v.hostNames (name: {
|
||||
targets = [ "${name}:${toString v.port}" ];
|
||||
labels.alias = name;
|
||||
});
|
||||
in
|
||||
(mkIf (static_configs != []) ({
|
||||
inherit static_configs;
|
||||
job_name = k;
|
||||
scrape_interval = "15s";
|
||||
} // (removeAttrs v [ "hostNames" "port" ]))));
|
||||
|
||||
prometheus = {
|
||||
systemd.services.prometheus.serviceConfig.LimitNOFILE = 1024000;
|
||||
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
|
||||
ruleFiles = singleton (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
|
||||
groups = singleton {
|
||||
name = "alerting-rules";
|
||||
rules = import ./alert-rules.nix { inherit lib; };
|
||||
};
|
||||
}));
|
||||
|
||||
scrapeConfigs = (mkScrapeConfigs ({
|
||||
node = {
|
||||
hostNames = cfg.server.scrapeHosts;
|
||||
port = 9100;
|
||||
};
|
||||
infiniband = {
|
||||
hostNames = [ "stokes" ];
|
||||
port = 9683;
|
||||
};
|
||||
slurm = {
|
||||
hostNames = [ "stokes" ];
|
||||
port = 6080;
|
||||
};
|
||||
}));
|
||||
};
|
||||
};
|
||||
|
||||
nodeExporter = {
|
||||
services.prometheus.exporters = {
|
||||
node = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
extraFlags = [ "--collector.disable-defaults" ];
|
||||
enabledCollectors = [
|
||||
"netstat"
|
||||
"stat"
|
||||
"systemd"
|
||||
"textfile"
|
||||
"textfile.directory /run/prometheus-node-exporter"
|
||||
"thermal_zone"
|
||||
"time"
|
||||
"udp_queues"
|
||||
"uname"
|
||||
"vmstat"
|
||||
"cpu"
|
||||
"cpufreq"
|
||||
"diskstats"
|
||||
"edac"
|
||||
"filesystem"
|
||||
"hwmon"
|
||||
"interrupts"
|
||||
"ksmd"
|
||||
"loadavg"
|
||||
"meminfo"
|
||||
"pressure"
|
||||
"timex"
|
||||
# "nfsd"
|
||||
# "nfs"
|
||||
# "rapl"
|
||||
] ++ cfg.nodeExporter.extraCollectors;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 9093 9100 ];
|
||||
};
|
||||
|
||||
webUI = let net = config.networking; in {
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
domain = "grafana.${net.domain}";
|
||||
port = 2342;
|
||||
addr = "127.0.0.1";
|
||||
};
|
||||
|
||||
security.acme = {
|
||||
acceptTerms = true;
|
||||
email = cfg.webUI.acmeEmail;
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 80 443 ];
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
|
||||
statusPage = true;
|
||||
|
||||
virtualHosts = {
|
||||
# "acme.${net.domain}" = {
|
||||
# serverAliases = [ "*.svc.${net.domain}" ];
|
||||
# # /var/lib/acme/.challenges must be writable by the ACME user
|
||||
# # and readable by the Nginx user.
|
||||
# locations."/.well-known/acme-challenge" = {
|
||||
# root = "/var/lib/acme/acme-challenge";
|
||||
# };
|
||||
# locations."/" = {
|
||||
# return = "301 https://$host$request_uri";
|
||||
# };
|
||||
# };
|
||||
|
||||
${config.services.grafana.domain} = {
|
||||
forceSSL = true;
|
||||
enableACME = true;
|
||||
serverAliases = [];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = webUIExtraConfig;
|
||||
};
|
||||
};
|
||||
|
||||
"prometheus.${net.domain}" = {
|
||||
forceSSL = true;
|
||||
enableACME = true;
|
||||
serverAliases = [];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = webUIExtraConfig;
|
||||
};
|
||||
};
|
||||
|
||||
"alertmanager.${net.domain}" = {
|
||||
forceSSL = true;
|
||||
enableACME = true;
|
||||
serverAliases = [];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString config.services.prometheus.alertmanager.port}";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = webUIExtraConfig;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
webUIExtraConfig =
|
||||
(builtins.foldl' (a: x:
|
||||
a + "\nallow ${x};" ) cfg.webUI.extraConfig cfg.webUI.allow)
|
||||
+ "\ndeny all;";
|
||||
|
||||
alertmanager = {
|
||||
systemd.services.alertmanager.serviceConfig.LimitNOFILE = 1024000;
|
||||
|
||||
services.prometheus.alertmanager = {
|
||||
enable = true;
|
||||
configuration = {
|
||||
route = {
|
||||
receiver = "default";
|
||||
routes = [
|
||||
{
|
||||
group_by = [ "alertname" "alias" ];
|
||||
group_wait = "5s";
|
||||
group_interval = "3m";
|
||||
repeat_interval = "3h";
|
||||
match = { severity = "page"; };
|
||||
receiver = "page";
|
||||
}
|
||||
{
|
||||
group_by = [ "alertname" "alias" ];
|
||||
group_wait = "30s";
|
||||
group_interval = "5m";
|
||||
repeat_interval = "6h";
|
||||
receiver = "default";
|
||||
}
|
||||
];
|
||||
};
|
||||
receivers = [
|
||||
({ name = "default"; } // cfg.server.defaultAlertReceiver)
|
||||
({ name = "page"; } // cfg.server.pageAlertReceiver)
|
||||
];
|
||||
inhibit_rules = [
|
||||
# {
|
||||
# target_match = {
|
||||
# alertname = "node_collector_failed";
|
||||
# };
|
||||
# target_match_re = {
|
||||
# alias = "c[0-9]-[0-9]";
|
||||
# collector = "nfsd";
|
||||
# };
|
||||
# }
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.prometheus = {
|
||||
alertmanagers = singleton {
|
||||
static_configs = singleton {
|
||||
targets = [ "localhost:9093" ];
|
||||
# targets = flip map cfg.server.scrapeHosts (n: "${n}:9093");
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in {
|
||||
options.features.monitoring = {
|
||||
server = {
|
||||
enable = mkEnableOption "HPC cluster monitoring server with prometheus";
|
||||
|
||||
scrapeHosts = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
defaultAlertReceiver = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
};
|
||||
|
||||
pageAlertReceiver = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
};
|
||||
};
|
||||
|
||||
nodeExporter.enable = mkEnableOption "Enable node exporter";
|
||||
|
||||
nodeExporter.extraCollectors = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
webUI = {
|
||||
enable = mkEnableOption "Enable web UI for monitoring";
|
||||
|
||||
acmeEmail = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
allow = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkMerge [
|
||||
(mkIf cfg.server.enable (mkMerge [
|
||||
prometheus
|
||||
alertmanager
|
||||
]))
|
||||
|
||||
(mkIf cfg.nodeExporter.enable nodeExporter)
|
||||
|
||||
(mkIf cfg.webUI.enable webUI)
|
||||
];
|
||||
|
||||
imports = [ ./infiniband-exporter.nix ./slurm-exporter.nix ];
|
||||
}
|
||||
67
modules/hpc/slurm-exporter.nix
Normal file
67
modules/hpc/slurm-exporter.nix
Normal file
@@ -0,0 +1,67 @@
|
||||
{pkgs, config, lib, ...}:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.features.monitoring.slurm-exporter;
|
||||
|
||||
slurm-exporter-service = {
|
||||
systemd.services.slurm-exporter = {
|
||||
enable = true;
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
description = "Prometheus SLURM exporter";
|
||||
script = ''
|
||||
PATH=$(${pkgs.nix}/bin/nix-store -q --referrers ${pkgs.slurm} | ${pkgs.gnugrep}/bin/grep wrappedSlurm)/bin
|
||||
${slurm-exporter}/bin/prometheus-slurm-exporter -listen-address :${builtins.toString cfg.port}
|
||||
'';
|
||||
serviceConfig = {
|
||||
RestartSec = "15s";
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
slurm-exporter = pkgs.buildGoModule rec {
|
||||
pname = "prometheus-slurm-exporter";
|
||||
version = "master";
|
||||
|
||||
src = pkgs.fetchFromGitHub {
|
||||
owner = "vpenso";
|
||||
repo = "prometheus-slurm-exporter";
|
||||
rev = "master";
|
||||
sha256 = "sha256-KS9LoDuLQFq3KoKpHd8vg1jw20YCNRJNJrnBnu5vxvs=";
|
||||
};
|
||||
|
||||
vendorHash = "sha256-A1dd9T9SIEHDCiVT2UwV6T02BSLh9ej6LC/2l54hgwI=";
|
||||
|
||||
nativeBuildInputs = [ pkgs.slurm ];
|
||||
|
||||
# subPackages = [ "." ];
|
||||
|
||||
# deleteVendor = true;
|
||||
|
||||
# runVend = true;
|
||||
|
||||
doCheck = false;
|
||||
|
||||
meta = with lib; {
|
||||
description = "Simple command-line snippet manager, written in Go";
|
||||
homepage = "https://github.com/knqyf263/pet";
|
||||
license = licenses.mit;
|
||||
maintainers = with maintainers; [ kalbasit ];
|
||||
platforms = platforms.linux ++ platforms.darwin;
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
options.features.monitoring.slurm-exporter = {
|
||||
enable = mkEnableOption "Enable SLURM prometheus exporter";
|
||||
|
||||
port = mkOption {
|
||||
type = types.ints.unsigned;
|
||||
default = 8080;
|
||||
description = "Collector http port";
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable slurm-exporter-service;
|
||||
}
|
||||
234
modules/hpc/slurm.nix
Normal file
234
modules/hpc/slurm.nix
Normal file
@@ -0,0 +1,234 @@
|
||||
{ pkgs, lib, config, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.features.hpc.slurm;
|
||||
|
||||
configuration = {
|
||||
services.munge.enable = true;
|
||||
environment.etc."munge/munge.key" = {
|
||||
source = cfg.mungeKey;
|
||||
mode = "0400";
|
||||
uid = cfg.mungeUid;
|
||||
gid = 0;
|
||||
};
|
||||
|
||||
services.slurm = {
|
||||
controlMachine = cfg.controlMachine;
|
||||
nodeName = cfg.nodeName;
|
||||
partitionName = cfg.partitionName;
|
||||
extraConfig = ''
|
||||
# AccountingStorageType=accounting_storage/none
|
||||
AccountingStorageType=accounting_storage/slurmdbd
|
||||
JobAcctGatherType=jobacct_gather/linux
|
||||
MailDomain=${cfg.mailDomain}
|
||||
MailProg=/run/wrappers/bin/sendmail
|
||||
SelectType=select/cons_tres
|
||||
SelectTypeParameters=CR_Core
|
||||
# AuthAltTypes=auth/jwt
|
||||
# AuthAltParameters=jwt_key=/var/spool/slurm/statesave/jwt_hs256.key
|
||||
'';
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 6818 ];
|
||||
|
||||
nixpkgs.overlays = [ slurm-ucx ];
|
||||
};
|
||||
|
||||
|
||||
slurmServer = {
|
||||
services.mysql = {
|
||||
enable = true;
|
||||
package = pkgs.mariadb;
|
||||
ensureUsers = [
|
||||
{
|
||||
name = "slurm";
|
||||
ensurePermissions = {
|
||||
"slurm_acct_db.*" = "ALL PRIVILEGES";
|
||||
};
|
||||
}
|
||||
];
|
||||
initialDatabases = [
|
||||
{ name = "slurm_acct_db"; }
|
||||
];
|
||||
};
|
||||
|
||||
services.slurm = {
|
||||
server.enable = true;
|
||||
# extraConfig = ''
|
||||
# MailDomain=itpartner.no
|
||||
# MailProg=${pkgs.ssmtp}/bin/ssmtp
|
||||
# '';
|
||||
dbdserver = {
|
||||
enable = true;
|
||||
# dbdHost = cfg.controlMachine;
|
||||
# storagePass = cfg.storagePass;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 6817 ];
|
||||
};
|
||||
|
||||
slurmClient = {
|
||||
services.slurm.client.enable = true;
|
||||
systemd.services.slurmd.serviceConfig = {
|
||||
Restart = "on-failure";
|
||||
};
|
||||
};
|
||||
|
||||
slurm-ucx = self: super: with super.pkgs; {
|
||||
slurm = super.slurm.overrideAttrs (attrs: {
|
||||
buildInputs = attrs.buildInputs ++ [ ucx http-parser pkg-config ];
|
||||
|
||||
nativeBuildInputs = attrs.nativeBuildInputs ++ [ makeWrapper ];
|
||||
|
||||
configureFlags =
|
||||
attrs.configureFlags ++ [
|
||||
"--with-ucx=${ucx.dev}"
|
||||
"--with-http-parser=${http-parser}"
|
||||
"--enable-slurmrestd"
|
||||
];
|
||||
|
||||
postFixup = ''
|
||||
wrapProgram $out/bin/slurmstepd --set LD_LIBRARY_PATH ${ucx}/lib
|
||||
wrapProgram $out/bin/srun --set SLURM_MPI_TYPE "pmix"
|
||||
'';
|
||||
# --set PSM3_PKEY "${cfg.pkey}" \
|
||||
# --set PMIX_MCA_gds "^ds12" \
|
||||
});
|
||||
};
|
||||
|
||||
hipster = {
|
||||
users.groups.hipster.gid = 2001;
|
||||
users.users.hipster = {
|
||||
description = "Job runner";
|
||||
home = "/work/hipster";
|
||||
group = "hipster";
|
||||
extraGroups = [
|
||||
"users"
|
||||
];
|
||||
uid = 2001;
|
||||
isNormalUser = true;
|
||||
createHome = false;
|
||||
useDefaultShell = true;
|
||||
};
|
||||
};
|
||||
|
||||
slurmrestd = {
|
||||
systemd.tmpfiles.rules = [ "d /run/slurmrestd 0750 hipster hipster -" ];
|
||||
|
||||
systemd.services.slurmrestd = {
|
||||
description = "Slurm REST API service";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "slurmd.service" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = "hipster";
|
||||
Group = "hipster";
|
||||
};
|
||||
environment = {
|
||||
# SLURM_JWT = "daemon";
|
||||
};
|
||||
script = ''
|
||||
rm -f /run/slurmrestd/hipster.socket
|
||||
/run/current-system/sw/bin/slurmrestd -v -a rest_auth/local unix:/run/slurmrestd/hipster.socket
|
||||
'';
|
||||
serviceConfig = {
|
||||
RuntimeDirectory = "slurmrestd";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.sockets.slurm-http-proxy = {
|
||||
enable = true;
|
||||
description = "Proxy slurmrestd unix socket to port 6822";
|
||||
listenStreams = [ "0.0.0.0:6822" ];
|
||||
wantedBy = [ "sockets.target" ];
|
||||
# Allow multiple instances of corresponding service.
|
||||
socketConfig.Accept = true;
|
||||
};
|
||||
|
||||
systemd.services."slurm-http-proxy@" = {
|
||||
enable = true;
|
||||
description = "Proxy slurmrestd unix socket to port 6822";
|
||||
serviceConfig = {
|
||||
ExecStart = "-${pkgs.socat}/bin/socat STDIO UNIX-CONNECT:/run/slurmrestd/hipster.socket";
|
||||
StandardInput="socket";
|
||||
User = "hipster";
|
||||
Group = "hipster";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.features.hpc.slurm = {
|
||||
enable = mkEnableOption "Enable SLURM batch system";
|
||||
|
||||
mungeKey = mkOption {
|
||||
type = types.path;
|
||||
default = null;
|
||||
};
|
||||
|
||||
mungeUid = mkOption {
|
||||
type = types.int;
|
||||
default = 997;
|
||||
};
|
||||
|
||||
pkey = mkOption {
|
||||
type = types.str;
|
||||
default = "0x7fff";
|
||||
};
|
||||
|
||||
controlMachine = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
server = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
client = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
hipster = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
};
|
||||
|
||||
slurmrestd = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
nodeName = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
partitionName = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [];
|
||||
};
|
||||
|
||||
storagePass = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
mailDomain = mkOption {
|
||||
type = types.str;
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable (
|
||||
mkMerge [
|
||||
configuration
|
||||
(mkIf cfg.server slurmServer)
|
||||
(mkIf cfg.client slurmClient)
|
||||
(mkIf (cfg.hipster) hipster)
|
||||
(mkIf (cfg.server && cfg.hipster) slurmrestd)
|
||||
(mkIf (cfg.slurmrestd && cfg.hipster) slurmrestd)
|
||||
]);
|
||||
}
|
||||
38
modules/hpc/xpmem.nix
Normal file
38
modules/hpc/xpmem.nix
Normal file
@@ -0,0 +1,38 @@
|
||||
{ pkgs, kernel ? pkgs.linux, ... } :
|
||||
with pkgs;
|
||||
let
|
||||
version = "master";
|
||||
kdir="${kernel.dev}/lib/modules/${kernel.modDirVersion}";
|
||||
in stdenv.mkDerivation {
|
||||
inherit version;
|
||||
name = "xpmem-${version}-${kernel.version}";
|
||||
|
||||
src = fetchgit {
|
||||
name = "xpmem-${version}";
|
||||
url = "https://github.com/hpc/xpmem.git";
|
||||
sha256 = "sha256-lB4adWBLhaj32Ll11jWPuCJSuKF8EVBG9L1Pk8HIbzY=";
|
||||
};
|
||||
|
||||
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
|
||||
|
||||
nativeBuildInputs = [ which kmod ];
|
||||
|
||||
buildInputs = kernel.moduleBuildDependencies ++ [
|
||||
libtool autoconf pkgconf automake hwloc
|
||||
] ;
|
||||
|
||||
preConfigurePhases = "preConfigure";
|
||||
preConfigure = ''
|
||||
autoupdate
|
||||
./autogen.sh
|
||||
'';
|
||||
|
||||
configureFlags = [
|
||||
"--with-kerneldir=${kdir}/source"
|
||||
];
|
||||
|
||||
installPhase = ''
|
||||
make install
|
||||
rm -rf $out/etc $out/sbin
|
||||
'';
|
||||
}
|
||||
Reference in New Issue
Block a user