155 lines
4.0 KiB
Nix
155 lines
4.0 KiB
Nix
{ pkgs ? import <nixpkgs> {} }:
|
|
let
|
|
# Pin the deployment package-set to a specific version of nixpkgs
|
|
# pkgs = import (builtins.fetchTarball {
|
|
# url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
|
|
# sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
|
|
# }) {};
|
|
# pkgs = import <nixpkgs> {};
|
|
etcdCluster = import ../etcdCluster.nix;
|
|
name = "fs-work";
|
|
address = "172.16.239.210";
|
|
in {
|
|
fs-work = { config, pkgs, ... }: with pkgs; {
|
|
deployment.tags = [ "fs" "fs-work" "cluster" ];
|
|
deployment.targetHost = address;
|
|
system.autoUpgrade.enable = lib.mkForce false;
|
|
|
|
systemd.targets = {
|
|
sleep.enable = false;
|
|
suspend.enable = false;
|
|
hibernate.enable = false;
|
|
hybrid-sleep.enable = false;
|
|
};
|
|
|
|
# services.udev.extraRules = ''
|
|
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
|
# '';
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
rdma-core
|
|
hwloc
|
|
xfsprogs
|
|
];
|
|
|
|
cluster = {
|
|
k8sNode = true;
|
|
slurm = false;
|
|
mounts = {
|
|
rdma.enable = false;
|
|
automount.enable = true;
|
|
users = false;
|
|
opt = false;
|
|
work = false;
|
|
data = false;
|
|
ceph = false;
|
|
};
|
|
};
|
|
|
|
features = {
|
|
host = {
|
|
inherit address;
|
|
inherit name;
|
|
};
|
|
|
|
os = {
|
|
networkmanager.enable = false;
|
|
externalInterface = "enp1s0np0";
|
|
nfs.enable = true;
|
|
nfs.exports = ''
|
|
/exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
|
'';
|
|
};
|
|
|
|
k8s = {
|
|
enable = true;
|
|
node.enable = true;
|
|
master.enable = false;
|
|
inherit etcdCluster;
|
|
};
|
|
};
|
|
|
|
systemd.services.rc-local = {
|
|
description = "rc.local script";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
path = [ "/run/current-system/sw/" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
};
|
|
script = ''
|
|
# if [ -e /sys/block/md126 ]; then
|
|
# echo "deadline" > /sys/block/md126/queue/scheduler
|
|
# # echo "4096" > /sys/block/md126/queue/nr_requests
|
|
# echo "4096" > /sys/block/md126/queue/read_ahead_kb
|
|
# echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
|
|
# echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
|
|
# fi
|
|
grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
|
|
grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
|
|
'';
|
|
};
|
|
|
|
boot.kernel.sysctl = {
|
|
"vm.dirty_background_ratio" = 5;
|
|
"vm.dirty_ratio" = 10;
|
|
"vm.vfs_cache_pressure" = 50;
|
|
"vm.min_free_kbytes" = 262144;
|
|
};
|
|
|
|
networking = {
|
|
useNetworkd = true;
|
|
useDHCP = false;
|
|
hostName = name;
|
|
defaultGateway = {
|
|
interface = "enp1s0np0";
|
|
address = "172.16.239.1";
|
|
};
|
|
firewall = {
|
|
allowedTCPPorts = [];
|
|
allowedUDPPorts = [];
|
|
extraCommands = ''
|
|
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j MASQUERADE
|
|
'';
|
|
};
|
|
};
|
|
|
|
systemd.network = {
|
|
networks."40-enp1s0np0" = {
|
|
matchConfig.Name = "enp1s0np0";
|
|
DHCP = "no";
|
|
address = [ "${address}/24" ];
|
|
routes = [
|
|
{ Gateway = "172.16.239.1"; }
|
|
{
|
|
Destination = "10.255.241.0/24";
|
|
Gateway = "172.16.239.221";
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
services.rpcbind.enable = true;
|
|
|
|
fileSystems = {
|
|
"/exports/work" = {
|
|
device = "/work";
|
|
options = [ "bind" ];
|
|
};
|
|
};
|
|
|
|
programs.singularity.enable = true;
|
|
|
|
boot.kernelParams = [
|
|
"amd_iommu=on" "iommu=pt"
|
|
];
|
|
boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_16;
|
|
|
|
imports = [
|
|
./hardware-configuration.nix
|
|
../default.nix
|
|
../mounts.nix
|
|
];
|
|
};
|
|
}
|