oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: add k8s and hpc modules to main repo

Browse Source
This commit is contained in:
Jonas Juselius
2025-06-30 12:21:05 +02:00
parent 4aa9fa677a
commit bc3a034654
46 changed files with 4393 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
modules/hpc/beegfs/beegfs.nix Normal file
View File

@@ -0,0 +1,107 @@
{ pkgs, kernel ? pkgs.linux, ... } :
with pkgs;
let
version = "7.4.0";
in stdenvNoCC.mkDerivation {
pname = "beegfs";
inherit version;
src = fetchurl {
name = "beegfs-archive-${version}.tar.bz2";
# url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}";
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2";
sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo=";
};
nativeBuildInputs = [ which unzip pkg-config cppunit perl makeWrapper ];
buildInputs = [
gcc12
libuuid
attr
xfsprogs
zlib
openssl
sqlite
rdma-core
openssh
gfortran
influxdb
curl
rdma-core
pahole
];
hardeningDisable = [ "format" ]; # required for building beeond
postPatch = ''
patchShebangs ./
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
'';
buildPhase = ''
make BEEGFS_OPENTK_IBVERBS=1 \
KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build \
''${enableParallelBuilding:+-j''${NIX_BUILD_CORES} \
-l''${NIX_BUILD_CORES}}
'';
enableParallelBuilding = true;
installPhase = ''
binDir=$out/bin
docDir=$out/share/doc/beegfs
includeDir=$out/include/beegfs
libDir=$out/lib
libDirPkg=$out/lib/beegfs
mkdir -p $binDir $libDir $libDirPkg $docDir $includeDir
cp common/build/libbeegfs_ib.so $libDir
cp ctl/build/beegfs-ctl $binDir
cp fsck/build/beegfs-fsck $binDir
cp utils/scripts/beegfs-check-servers $binDir
cp utils/scripts/beegfs-df $binDir
cp utils/scripts/beegfs-net $binDir
cp helperd/build/beegfs-helperd $binDir
cp helperd/build/dist/etc/beegfs-helperd.conf $docDir
cp client_module/build/dist/sbin/beegfs-setup-client $binDir
cp client_module/build/dist/etc/beegfs-client.conf $docDir
cp meta/build/beegfs-meta $binDir
cp meta/build/dist/sbin/beegfs-setup-meta $binDir
cp meta/build/dist/etc/beegfs-meta.conf $docDir
cp mgmtd/build/beegfs-mgmtd $binDir
cp mgmtd/build/dist/sbin/beegfs-setup-mgmtd $binDir
cp mgmtd/build/dist/etc/beegfs-mgmtd.conf $docDir
cp storage/build/beegfs-storage $binDir
cp storage/build/dist/sbin/beegfs-setup-storage $binDir
cp storage/build/dist/etc/beegfs-storage.conf $docDir
cp client_devel/build/dist/usr/share/doc/beegfs-client-devel/examples/* $docDir
cp -r client_devel/include/* $includeDir
'';
# postFixup = ''
# for i in $(find $out/bin -type f -executable); do
# wrapProgram "$i" --prefix LD_LIBRARY_PATH : $out/lib
# done
# '';
doCheck = true;
# checkPhase = ''
# LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/opentk_lib/build/ \
# common/build/test-runner --text
# '';
meta = with lib; {
description = "High performance distributed filesystem with RDMA support";
homepage = "https://www.beegfs.io";
platforms = [ "i686-linux" "x86_64-linux" ];
license = {
fullName = "BeeGFS_EULA";
url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
free = true;
};
maintainers = with maintainers; [ "juselius" ];
};
}

340
modules/hpc/beegfs/default.nix Normal file
View File

@@ -0,0 +1,340 @@
{ config, lib, pkgs, ...} :
with lib;
let
cfg = config.features.hpc.beegfs.beegfs;
# kernel = pkgs.linuxPackages_5_4.kernel;
kernel = config.boot.kernelPackages.kernel;
beegfs = pkgs.callPackage ./beegfs.nix {
inherit kernel;
};
beegfs-module = pkgs.callPackage ./kernel-module.nix {
inherit kernel;
};
# functions for the generations of config files
configMgmtd = name: cfg: pkgs.writeText "mgmt-${name}.conf" ''
storeMgmtdDirectory = ${cfg.mgmtd.storeDir}
storeAllowFirstRunInit = false
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
${cfg.mgmtd.extraConfig}
'';
configMeta = name: cfg: pkgs.writeText "meta-${name}.conf" ''
storeMetaDirectory = ${cfg.meta.storeDir}
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
storeAllowFirstRunInit = false
${cfg.meta.extraConfig}
'';
configStorage = name: cfg: pkgs.writeText "storage-${name}.conf" ''
storeStorageDirectory = ${cfg.storage.storeDir}
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
storeAllowFirstRunInit = false
${cfg.storage.extraConfig}
'';
configHelperd = name: cfg: pkgs.writeText "helperd-${name}.conf" ''
connAuthFile = ${cfg.connAuthFile}
${cfg.helperd.extraConfig}
'';
configClientFilename = name : "/etc/beegfs/client-${name}.conf";
configClient = name: cfg: ''
sysMgmtdHost = ${cfg.mgmtdHost}
connAuthFile = ${cfg.connAuthFile}
connPortShift = ${toString cfg.connPortShift}
${cfg.client.extraConfig}
'';
serviceList = [
{ service = "meta"; cfgFile = configMeta; }
{ service = "mgmtd"; cfgFile = configMgmtd; }
{ service = "storage"; cfgFile = configStorage; }
];
# functions to generate systemd.service entries
systemdEntry = service: cfgFile: (mapAttrs' ( name: cfg:
(nameValuePair "beegfs-${service}-${name}" (mkIf cfg.${service}.enable {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
serviceConfig = rec {
ExecStart = ''
${beegfs}/bin/beegfs-${service} \
cfgFile=${cfgFile name cfg} \
pidFile=${PIDFile}
'';
PIDFile = "/run/beegfs-${service}-${name}.pid";
TimeoutStopSec = "300";
};
}))) cfg);
systemdHelperd = mapAttrs' ( name: cfg:
(nameValuePair "beegfs-helperd-${name}" (mkIf cfg.client.enable {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
environment = {
LD_LIBRARY_PATH = "${beegfs}/lib";
};
serviceConfig = rec {
ExecStart = ''
${beegfs}/bin/beegfs-helperd \
cfgFile=${configHelperd name cfg} \
pidFile=${PIDFile}
'';
PIDFile = "/run/beegfs-helperd-${name}.pid";
TimeoutStopSec = "300";
};
}))) cfg;
# wrappers to beegfs tools. Avoid typing path of config files
utilWrappers = mapAttrsToList ( name: cfg:
( pkgs.runCommand "beegfs-utils-${name}" {
nativeBuildInputs = [ pkgs.makeWrapper ];
preferLocalBuild = true;
} ''
mkdir -p $out/bin
makeWrapper ${beegfs}/bin/beegfs-check-servers \
$out/bin/beegfs-check-servers-${name} \
--add-flags "-c ${configClientFilename name}" \
--prefix PATH : ${lib.makeBinPath [ beegfs ]}
makeWrapper ${beegfs}/bin/beegfs-ctl \
$out/bin/beegfs-ctl-${name} \
--add-flags "--cfgFile=${configClientFilename name}"
makeWrapper ${beegfs}/bin/beegfs-ctl \
$out/bin/beegfs-df-${name} \
--add-flags "--cfgFile=${configClientFilename name}" \
--add-flags --listtargets \
--add-flags --hidenodeid \
--add-flags --pools \
--add-flags --spaceinfo
makeWrapper ${beegfs}/bin/beegfs-fsck \
$out/bin/beegfs-fsck-${name} \
--add-flags "--cfgFile=${configClientFilename name}"
''
)) cfg;
beegfsOptions = {
options = {
mgmtdHost = mkOption {
type = types.str;
default = null;
example = "master";
description = ''Hostname of managament host.'';
};
connAuthFile = mkOption {
type = types.str;
default = "";
example = "/etc/my.key";
description = "File containing shared secret authentication.";
};
connPortShift = mkOption {
type = types.int;
default = 0;
example = 5;
description = ''
For each additional beegfs configuration shift all
service TCP/UDP ports by at least 5.
'';
};
client = {
enable = mkEnableOption "BeeGFS client";
mount = mkOption {
type = types.bool;
default = true;
description = "Create fstab entry automatically";
};
mountPoint = mkOption {
type = types.str;
default = "/run/beegfs";
description = ''
Mount point under which the beegfs filesytem should be mounted.
If mounted manually the mount option specifing the config file is needed:
cfgFile=/etc/beegfs/beegfs-client-<name>.conf
'';
};
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-client.conf.
See documentation for further details.
'';
};
};
helperd = {
enable = mkOption {
type = types.bool;
default = true;
description = ''
Enable the BeeGFS helperd.
The helpered is need for logging purposes on the client.
Disabling <literal>helperd</literal> allows for runing the client
with <literal>allowUnfree = false</literal>.
'';
};
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-helperd.conf. See documentation
for further details.
'';
};
};
mgmtd = {
enable = mkEnableOption "BeeGFS mgmtd daemon";
storeDir = mkOption {
type = types.path;
default = null;
example = "/data/beegfs-mgmtd";
description = ''
Data directory for mgmtd.
Must not be shared with other beegfs daemons.
This directory must exist and it must be initialized
with beegfs-setup-mgmtd, e.g. "beegfs-setup-mgmtd -C -p &lt;storeDir&gt;"
'';
};
extraConfig = mkOption {
type = types.lines;
default = "";
description = ''
Additional lines for beegfs-mgmtd.conf. See documentation
for further details.
'';
};
};
meta = {
enable = mkEnableOption "BeeGFS meta data daemon";
storeDir = mkOption {
type = types.path;
default = null;
example = "/data/beegfs-meta";
description = ''
Data directory for meta data service.
Must not be shared with other beegfs daemons.
The underlying filesystem must be mounted with xattr turned on.
This directory must exist and it must be initialized
with beegfs-setup-meta, e.g.
"beegfs-setup-meta -C -s &lt;serviceID&gt; -p &lt;storeDir&gt;"
'';
};
extraConfig = mkOption {
type = types.str;
default = "";
description = ''
Additional lines for beegfs-meta.conf. See documentation
for further details.
'';
};
};
storage = {
enable = mkEnableOption "BeeGFS storage daemon";
storeDir = mkOption {
type = types.path;
default = null;
example = "/data/beegfs-storage";
description = ''
Data directories for storage service.
Must not be shared with other beegfs daemons.
The underlying filesystem must be mounted with xattr turned on.
This directory must exist and it must be initialized
with beegfs-setup-storage, e.g.
"beegfs-setup-storage -C -s &lt;serviceID&gt; -i &lt;storageTargetID&gt; -p &lt;storeDir&gt;"
'';
};
extraConfig = mkOption {
type = types.str;
default = "";
description = ''
Addional lines for beegfs-storage.conf. See documentation
for further details.
'';
};
};
};
};
in
{
###### interface
options.features.hpc.beegfs = {
enable = mkEnableOption "BeeGFS";
beegfs = mkOption {
type = with types; attrsOf (submodule ({ ... } : beegfsOptions ));
default = {};
description = ''
BeeGFS configurations. Every mount point requires a separate configuration.
'';
};
};
###### implementation
config = mkIf config.features.hpc.beegfs.enable {
environment.systemPackages = utilWrappers;
# Put the client.conf files in /etc since they are needed
# by the commandline tools
environment.etc = mapAttrs' ( name: cfg:
(nameValuePair "beegfs/client-${name}.conf" (mkIf (cfg.client.enable)
{
enable = true;
text = configClient name cfg;
}))) cfg;
# Kernel module, we need it only once per host.
boot = mkIf (
foldr (a: b: a || b) false
(map (x: x.client.enable) (collect (x: x ? client) cfg)))
{
kernelModules = [ "beegfs" ];
extraModulePackages = [ beegfs-module ];
};
# generate fstab entries
fileSystems = mapAttrs' (name: cfg:
(nameValuePair cfg.client.mountPoint (optionalAttrs cfg.client.mount (mkIf cfg.client.enable {
device = "beegfs_nodev";
fsType = "beegfs";
mountPoint = cfg.client.mountPoint;
options = [ "cfgFile=${configClientFilename name}" "_netdev" ];
})))) cfg;
# generate systemd services
systemd.services = systemdHelperd //
foldr (a: b: a // b) {}
(map (x: systemdEntry x.service x.cfgFile) serviceList);
};
}

50
modules/hpc/beegfs/kernel-module.nix Normal file
View File

@@ -0,0 +1,50 @@
{ pkgs, kernel ? pkgs.linux, ... } :
with pkgs;
let
version = "7.4.0";
beegfs = pkgs.callPackage ./beegfs.nix { inherit kernel; };
in stdenvNoCC.mkDerivation {
name = "beegfs-module-${version}-${kernel.version}";
src = fetchurl {
name = "beegfs-archive-${version}.tar.bz2";
# url = "https://git.beegfs.com/pub/v7/repository/archive.tar.bz2?ref=${version}";
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.bz2";
sha256 = "sha256-VwD3z3lZIs5aOIBbwUvEkOxkFggTCv8OWuJMCga2ooo=";
};
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
nativeBuildInputs = [ gcc12 which kmod pahole ];
buildInputs = kernel.moduleBuildDependencies;
makeFlags = [ "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/" ];
postPatch = ''
patchShebangs ./
find -type f -name Makefile -exec sed -i "s:/bin/bash:${stdenv.shell}:" \{} \;
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "configure" -exec sed -i "s:/bin/:/usr/bin/env :" \{} \;
find -type f -name "configure" -exec sed -i "s:/usr/bin/:/usr/bin/env :" \{} \;
sed -i 's,libbeegfs_ib.so,${beegfs}/lib/&,' common/source/common/net/sock/RDMASocket.cpp
'';
preBuild = "cd client_module/build";
installPhase = ''
instdir=$out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
mkdir -p $instdir
cp beegfs.ko $instdir
'';
meta = with lib; {
description = "High performance distributed filesystem with RDMA support";
homepage = "https://www.beegfs.io";
platforms = [ "i686-linux" "x86_64-linux" ];
license = licenses.gpl2;
maintainers = with maintainers; [ "juselius" ];
# broken = stdenv.lib.versionAtLeast kernel.version "4.18";
};
}