342 lines
10 KiB
Nix
342 lines
10 KiB
Nix
{ pkgs, lib, config, ... }:
|
|
with lib;
|
|
let
|
|
cfg = config.features.host;
|
|
|
|
etcdCluster = {
|
|
enable = true;
|
|
existing = true;
|
|
nodes =
|
|
{
|
|
c0-0 = "10.1.61.100";
|
|
c0-1 = "10.1.61.101";
|
|
c0-2 = "10.1.61.102";
|
|
};
|
|
};
|
|
|
|
mkSANs = host: [
|
|
host.name
|
|
host.address
|
|
"127.0.0.1"
|
|
];
|
|
|
|
configuration = {
|
|
system.autoUpgrade.enable = lib.mkForce false;
|
|
|
|
nixpkgs.overlays = [
|
|
(import ./overlays.nix)
|
|
];
|
|
|
|
boot = {
|
|
loader.systemd-boot.enable = true;
|
|
loader.efi.canTouchEfiVariables = true;
|
|
# kernelPackages = pkgs.linuxPackages_5_4;
|
|
kernelModules = [ "ib_umad" "ib_ipoib" ];
|
|
kernelParams = [
|
|
"console=ttyS0,115200"
|
|
"console=tty0"
|
|
];
|
|
};
|
|
|
|
console = {
|
|
font = "Lat2-Terminus16";
|
|
keyMap = "us";
|
|
};
|
|
|
|
i18n = {
|
|
defaultLocale = "en_US.UTF-8";
|
|
extraLocaleSettings = {
|
|
LC_CTYPE="en_DK.UTF-8";
|
|
LC_TIME="en_DK.UTF-8";
|
|
LC_PAPER="en_DK.UTF-8";
|
|
LC_NAME="en_DK.UTF-8";
|
|
LC_ADDRESS="en_DK.UTF-8";
|
|
LC_TELEPHONE="en_DK.UTF-8";
|
|
LC_MEASUREMENT="en_DK.UTF-8";
|
|
LC_IDENTIFICATION="en_DK.UTF-8";
|
|
};
|
|
};
|
|
|
|
time.timeZone = "Europe/Oslo";
|
|
|
|
programs.msmtp = {
|
|
enable = true;
|
|
accounts = {
|
|
default = {
|
|
auth = false;
|
|
tls = false;
|
|
tls_starttls = false;
|
|
port = 24;
|
|
from = "stokes@regnekraft.io";
|
|
host = "smtpgw.itpartner.no";
|
|
# user = "utvikling";
|
|
# password = "S0m3rp0m@de#21!";
|
|
};
|
|
};
|
|
defaults = {
|
|
aliases = "/etc/aliases";
|
|
};
|
|
};
|
|
|
|
environment.etc = {
|
|
"aliases" = {
|
|
text = ''
|
|
root: jonas.juselius@oceanbox.io
|
|
'';
|
|
mode = "0644";
|
|
};
|
|
};
|
|
|
|
features = {
|
|
os = {
|
|
# boot.uefi = true;
|
|
adminAuthorizedKeys = [
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas"
|
|
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
|
|
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
|
|
];
|
|
docker.enable = false;
|
|
};
|
|
cachix.enable = false;
|
|
|
|
monitoring.nodeExporter.enable = false;
|
|
|
|
pki = { ca = ./ca; };
|
|
|
|
hpc = {
|
|
enable = true;
|
|
slurm = {
|
|
client = true;
|
|
mungeKey = ./munge.key;
|
|
mungeUid = 997; # hack
|
|
controlMachine = "stokes";
|
|
nodeName = [
|
|
"c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
|
|
"stokes Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=64000 TmpDisk=500000 State=UNKNOWN"
|
|
];
|
|
partitionName = [
|
|
"batch Nodes=c0-[1-8] Default=YES MaxTime=INFINITE State=UP"
|
|
"frontend Nodes=stokes MaxTime=1:00:00 State=UP"
|
|
];
|
|
};
|
|
beegfs = {
|
|
enable = true;
|
|
beegfs = {
|
|
mds0-0 = {
|
|
mgmtdHost = "mds0-0";
|
|
connAuthFile = "/etc/beegfs/connauthfile";
|
|
client = {
|
|
enable = true;
|
|
mountPoint = "/work";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
k8s = {
|
|
enable = true;
|
|
node.enable = true;
|
|
clusterName = "hpc0";
|
|
inherit etcdCluster;
|
|
initca = ./ca;
|
|
cidr = "10.100.0.0/16";
|
|
master = {
|
|
name = "stokes";
|
|
address = "10.1.61.100";
|
|
extraSANs = [ "hpc0-0.regnekraft.io" ];
|
|
};
|
|
ingressNodes = [
|
|
"hpc0-0.regnekraft.io"
|
|
];
|
|
fileserver = "mds0-0";
|
|
charts = {
|
|
acme_email = "innovasjon@itpartner.no";
|
|
grafana_smtp_user = "utvikling";
|
|
grafana_smtp_password = "S0m3rp0m@de#21!";
|
|
};
|
|
};
|
|
};
|
|
|
|
services.kubernetes.kubelet.extraSANs = mkSANs {
|
|
name = cfg.name;
|
|
address = cfg.address;
|
|
};
|
|
|
|
networking = {
|
|
domain = mkDefault "regnekraft.io";
|
|
defaultGateway = mkDefault "10.1.61.1";
|
|
nameservers = mkDefault [ "8.8.8.8" ];
|
|
search = mkDefault [ "local" ];
|
|
extraHosts = import ./hosts.nix;
|
|
firewall.extraCommands = ''
|
|
iptables -I INPUT -s 10.1.61.0/24 -j ACCEPT
|
|
'';
|
|
};
|
|
|
|
fileSystems = {
|
|
"/opt" = {
|
|
device = "10.1.63.80:/opt";
|
|
fsType = "nfs";
|
|
options = [ "soft" "rdma" "defaults" "vers=4.2" ];
|
|
};
|
|
"/data" = {
|
|
device = "10.1.63.80:/data";
|
|
fsType = "nfs";
|
|
options = [ "soft" "rdma" "defaults" "vers=4.2" ];
|
|
};
|
|
};
|
|
|
|
environment.variables = {};
|
|
|
|
systemd.services."serial-getty@ttyS0".enable = true;
|
|
|
|
environment.etc."beegfs/connauthfile" = {
|
|
source = ./connauthfile;
|
|
mode = "0400";
|
|
uid = 0;
|
|
gid = 0;
|
|
};
|
|
|
|
nix = {
|
|
maxJobs = 32;
|
|
trustedUsers = [ "@wheel" ];
|
|
# binaryCachePublicKeys = [
|
|
# "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig=="
|
|
# ];
|
|
};
|
|
};
|
|
|
|
i40efix = {
|
|
# boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
|
|
# kernel = pkgs.linuxPackages_5_4.kernel;
|
|
# }; in {
|
|
# extraModulePackages = [ kernelExtras.i40e2 ];
|
|
# kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
|
|
# };
|
|
};
|
|
|
|
shosts = {
|
|
environment.etc."ssh/shosts.equiv" = {
|
|
mode = "0644";
|
|
uid = 0;
|
|
gid = 0;
|
|
text = ''
|
|
10.1.62.2
|
|
10.1.61.100
|
|
10.1.61.101
|
|
10.1.61.102
|
|
10.1.61.103
|
|
10.1.61.104
|
|
10.1.61.105
|
|
10.1.61.106
|
|
10.1.61.107
|
|
10.1.61.108
|
|
10.1.63.100
|
|
10.1.63.101
|
|
10.1.63.102
|
|
10.1.63.103
|
|
10.1.63.104
|
|
10.1.63.105
|
|
10.1.63.106
|
|
10.1.63.107
|
|
10.1.63.108
|
|
'';
|
|
};
|
|
|
|
programs.ssh.knownHosts = {
|
|
stokes = {
|
|
hostNames = [
|
|
"stokes" "stokes.hpc.local" "stokes.regnekraft.io" "10.1.61.100"
|
|
"10.1.63.100" "10.1.62.2"
|
|
];
|
|
publicKeyFile = ./pubkeys/stokes.pub;
|
|
};
|
|
c0-1 = { hostNames = [ "c0-1" "c0-1.hpc.local" "10.1.61.101" "10.1.63.101" ]; publicKeyFile = ./pubkeys/c0-1.pub; };
|
|
c0-2 = { hostNames = [ "c0-2" "c0-2.hpc.local" "10.1.61.102" "10.1.63.102" ]; publicKeyFile = ./pubkeys/c0-2.pub; };
|
|
c0-3 = { hostNames = [ "c0-3" "c0-3.hpc.local" "10.1.61.103" "10.1.63.103" ]; publicKeyFile = ./pubkeys/c0-3.pub; };
|
|
c0-4 = { hostNames = [ "c0-4" "c0-4.hpc.local" "10.1.61.104" "10.1.63.104" ]; publicKeyFile = ./pubkeys/c0-4.pub; };
|
|
c0-5 = { hostNames = [ "c0-5" "c0-5.hpc.local" "10.1.61.105" "10.1.63.105" ]; publicKeyFile = ./pubkeys/c0-5.pub; };
|
|
c0-6 = { hostNames = [ "c0-6" "c0-6.hpc.local" "10.1.61.106" "10.1.63.106" ]; publicKeyFile = ./pubkeys/c0-6.pub; };
|
|
c0-7 = { hostNames = [ "c0-7" "c0-7.hpc.local" "10.1.61.107" "10.1.63.107" ]; publicKeyFile = ./pubkeys/c0-7.pub; };
|
|
c0-8 = { hostNames = [ "c0-8" "c0-8.hpc.local" "10.1.61.108" "10.1.63.108" ]; publicKeyFile = ./pubkeys/c0-8.pub; };
|
|
};
|
|
|
|
environment.systemPackages = [ openssh-shosts ];
|
|
|
|
security.wrappers = {
|
|
ssh-keysign = {
|
|
source = "${openssh-shosts}/libexec/ssh-keysign";
|
|
owner = "root";
|
|
group = "root";
|
|
permissions = "u+rs,g+rx,o+rx";
|
|
};
|
|
};
|
|
};
|
|
|
|
openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {
|
|
buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ];
|
|
doCheck = false; # the tests take hours
|
|
});
|
|
|
|
myvnc =
|
|
let
|
|
myvnc = pkgs.writeScriptBin "myvnc" ''
|
|
#!${pkgs.runtimeShell}
|
|
|
|
uid=`id -u`
|
|
port=$((9000+$uid))
|
|
shell=`getent passwd $(id -un) | awk -F : '{print $NF}'`
|
|
# vnc=${pkgs.tigervnc}/bin/vncserver
|
|
vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver
|
|
|
|
case $1 in
|
|
-p|--port) shift; port=$1 ;;
|
|
kill|stop)
|
|
display=$($vnc -list | sed -n 's/^\(:[0-9]\+\).*/\1/p'| head -1)
|
|
$vnc -kill $display
|
|
exit 0
|
|
;;
|
|
esac
|
|
ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1
|
|
[ $? = 1 ] && $vnc -rfbport $port
|
|
echo "Xvnc server is running on port $port."
|
|
exec $shell -i
|
|
'';
|
|
|
|
buildCommand = ''
|
|
mkdir -p $out/bin
|
|
echo $src > $out/bin/myvnc
|
|
chmod 755 $out/bin/myvnc
|
|
'';
|
|
in {
|
|
environment.systemPackages = [ myvnc ];
|
|
};
|
|
|
|
in {
|
|
options.node = {
|
|
# i40efix = mkEnableOption "Apply fix for i40e driver";
|
|
|
|
myvnc = mkEnableOption "Enable myvnc script";
|
|
};
|
|
|
|
config = mkMerge [
|
|
configuration
|
|
|
|
# deployment
|
|
|
|
shosts
|
|
|
|
# (mkIf config.node.i40efix i40efix)
|
|
|
|
(mkIf config.node.myvnc myvnc)
|
|
];
|
|
|
|
imports = [
|
|
../../modules
|
|
../../nixos
|
|
./users.nix
|
|
];
|
|
}
|
|
|