137 lines
3.5 KiB
Nix
137 lines
3.5 KiB
Nix
{
|
|
pkgs ? import <nixpkgs> { },
|
|
}:
|
|
let
|
|
# Pin the deployment package-set to a specific version of nixpkgs
|
|
# pkgs = import (builtins.fetchTarball {
|
|
# url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
|
|
# sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
|
|
# }) {};
|
|
# pkgs = import <nixpkgs> {};
|
|
|
|
nodes = import ./nodes.nix;
|
|
has100GbE = [ "c0-18" ];
|
|
|
|
mkCompute =
|
|
host:
|
|
let
|
|
hw = ./hardware-configuration.d + "/${host.name}.nix";
|
|
in
|
|
{
|
|
"${host.name}" = {
|
|
deployment.tags = [
|
|
"compute"
|
|
"c0"
|
|
"cluster"
|
|
];
|
|
deployment.targetHost = host.address;
|
|
|
|
cluster = {
|
|
compute = true;
|
|
k8sNode = true;
|
|
mounts = {
|
|
rdma.enable = true;
|
|
automount.enable = true;
|
|
users = true;
|
|
opt = true;
|
|
work = true;
|
|
data = false;
|
|
ceph = false;
|
|
backup = false;
|
|
}
|
|
// (
|
|
if (builtins.elem host.name has100GbE) then
|
|
{
|
|
data = true;
|
|
ceph = true;
|
|
}
|
|
else
|
|
{ }
|
|
);
|
|
};
|
|
|
|
features = {
|
|
host = {
|
|
name = host.name;
|
|
address = host.address;
|
|
};
|
|
os.networkmanager.enable = false;
|
|
os.externalInterface = host.iface;
|
|
hpc.computeNode = true;
|
|
hpc.knem = true;
|
|
};
|
|
|
|
# services.udev.extraRules = ''
|
|
# KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
|
# '';
|
|
|
|
networking = {
|
|
useNetworkd = true;
|
|
hostName = host.name;
|
|
useDHCP = false;
|
|
};
|
|
|
|
# systemd.services.systemd-networkd-wait-online.enable = false;
|
|
|
|
systemd.network = {
|
|
wait-online.ignoredInterfaces = [ "ibp1s0" ];
|
|
# wait-online.enable = false;
|
|
networks = {
|
|
"40-${host.iface}" = {
|
|
matchConfig.Name = host.iface;
|
|
address = [ "${host.address}/24" ];
|
|
# networkConfig = {
|
|
# DNSDefaultRoute = true;
|
|
# };
|
|
routes = [
|
|
{ Gateway = "10.255.241.1"; }
|
|
{
|
|
Destination = "172.16.239.0/24";
|
|
Gateway = "10.255.241.210";
|
|
}
|
|
{
|
|
Destination = "10.255.242.0/24";
|
|
Gateway = "10.255.241.100";
|
|
}
|
|
];
|
|
};
|
|
"45-ibp1s0" = {
|
|
matchConfig.Name = "ibp1s0";
|
|
address = [ "${host.ipoib}/24" ];
|
|
};
|
|
}
|
|
// (
|
|
if (builtins.elem host.name has100GbE) then
|
|
{
|
|
"42-enp65s0f1np1" = {
|
|
DHCP = "no";
|
|
matchConfig.Name = "enp65s0f1np1 ";
|
|
address = [ "${host.gbe100}/24" ];
|
|
};
|
|
}
|
|
else
|
|
{ }
|
|
);
|
|
};
|
|
|
|
boot.kernelParams = [
|
|
"console=tty0"
|
|
"console=ttyS0,115200"
|
|
];
|
|
systemd.services."serial-getty@ttyS0" = {
|
|
enable = true;
|
|
wantedBy = [ "getty.target" ];
|
|
serviceConfig.Restart = "always";
|
|
};
|
|
|
|
imports = [
|
|
hw
|
|
../default.nix
|
|
../mounts.nix
|
|
./kernel.nix
|
|
];
|
|
};
|
|
};
|
|
in
|
|
builtins.foldl' (a: n: a // mkCompute n) { } nodes
|