{ pkgs, lib, config, ... }: with lib; let cfg = config.features.host; mkSANs = host: [ host.name host.address "127.0.0.1" ]; configuration = { system.autoUpgrade.enable = lib.mkForce false; boot = { loader.systemd-boot.enable = true; loader.efi.canTouchEfiVariables = true; kernelPackages = pkgs.linuxPackages_5_4; kernelModules = [ "ib_umad" "ib_ipoib" ]; kernelParams = [ "console=ttyS0,115200" "console=tty0" ]; }; console = { font = "Lat2-Terminus16"; keyMap = "us"; }; i18n = { defaultLocale = "en_DK.UTF-8"; extraLocaleSettings = { LC_TIME = "en_DK.UTF-8"; }; }; time.timeZone = "Europe/Oslo"; features = { os = { # boot.uefi = true; adminAuthorizedKeys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no" ]; docker.enable = false; mailRelay = { enable = true; adminEmail = "jonas.juselius@tromso.serit.no"; mailDomain = "itpartner.no"; mailGateway = "smtpgw.itpartner.no:465"; mailAuthUser = "utvikling"; }; }; cachix.enable = false; monitoring.nodeExporter.enable = false; pki = { ca = ./ca; }; hpc = { enable = true; slurm = { client = true; mungeKey = ./munge.key; controlMachine = "stokes"; nodeName = [ "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" "stokes Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=64000 TmpDisk=500000 State=UNKNOWN" ]; partitionName = [ "batch Nodes=c0-[1-8] Default=YES MaxTime=INFINITE State=UP" "frontend Nodes=stokes MaxTime=1:00:00 State=UP" ]; }; beegfs = { enable = true; beegfs = { mds0-0 = { mgmtdHost = "mds0-0"; connAuthFile = ""; client = { enable = true; mountPoint = "/work"; }; }; }; }; }; k8s = { enable = true; node.enable = true; clusterName = "hpc0"; initca = ./ca; cidr = "10.100.0.0/16"; master = { name = "stokes"; address = "10.1.61.100"; extraSANs = [ "hpc0-0.regnekraft.io" ]; }; ingressNodes = [ "hpc0-0.regnekraft.io" ]; fileserver = "mds0-0"; charts = { acme_email = "innovasjon@itpartner.no"; grafana_smtp_user = "utvikling"; grafana_smtp_password = "S0m3rp0m@de#21!"; }; }; }; services.kubernetes.kubelet.extraSANs = mkSANs { name = cfg.name; address = cfg.address; }; networking = { domain = mkDefault "regnekraft.io"; defaultGateway = mkDefault "10.1.61.1"; nameservers = mkDefault [ "8.8.8.8" ]; search = mkDefault [ "local" ]; extraHosts = import ./hosts.nix; firewall.extraCommands = '' iptables -I INPUT -s 10.1.61.0/24 -j ACCEPT ''; }; environment.variables = {}; systemd.services."serial-getty@ttyS0".enable = true; nix = { maxJobs = 32; trustedUsers = [ "@wheel" ]; # binaryCachePublicKeys = [ # "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig==" # ]; }; }; deployment = { deployment.targetHost = cfg.address; }; i40efix = { boot = let kernelExtras = pkgs.callPackage ./kernel.nix { kernel = pkgs.linuxPackages_5_4.kernel; }; in { extraModulePackages = [ kernelExtras.i40e2 ]; kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ]; }; }; shosts = { environment.etc."ssh/shosts.equiv" = { mode = "0644"; uid = 0; gid = 0; text = '' 10.1.62.2 10.1.61.100 10.1.61.101 10.1.61.102 10.1.61.103 10.1.61.104 10.1.61.105 10.1.61.106 10.1.61.107 10.1.61.108 10.1.63.100 10.1.63.101 10.1.63.102 10.1.63.103 10.1.63.104 10.1.63.105 10.1.63.106 10.1.63.107 10.1.63.108 ''; }; programs.ssh.knownHosts = { stokes = { hostNames = [ "stokes" "stokes.hpc.local" "stokes.regnekraft.io" "10.1.61.100" "10.1.63.100" "10.1.62.2" ]; publicKeyFile = ./pubkeys/stokes.pub; }; c0-1 = { hostNames = [ "c0-1" "c0-1.hpc.local" "10.1.61.101" "10.1.63.101" ]; publicKeyFile = ./pubkeys/c0-1.pub; }; c0-2 = { hostNames = [ "c0-2" "c0-2.hpc.local" "10.1.61.102" "10.1.63.102" ]; publicKeyFile = ./pubkeys/c0-2.pub; }; c0-3 = { hostNames = [ "c0-3" "c0-3.hpc.local" "10.1.61.103" "10.1.63.103" ]; publicKeyFile = ./pubkeys/c0-3.pub; }; c0-4 = { hostNames = [ "c0-4" "c0-4.hpc.local" "10.1.61.104" "10.1.63.104" ]; publicKeyFile = ./pubkeys/c0-4.pub; }; c0-5 = { hostNames = [ "c0-5" "c0-5.hpc.local" "10.1.61.105" "10.1.63.105" ]; publicKeyFile = ./pubkeys/c0-5.pub; }; c0-6 = { hostNames = [ "c0-6" "c0-6.hpc.local" "10.1.61.106" "10.1.63.106" ]; publicKeyFile = ./pubkeys/c0-6.pub; }; c0-7 = { hostNames = [ "c0-7" "c0-7.hpc.local" "10.1.61.107" "10.1.63.107" ]; publicKeyFile = ./pubkeys/c0-7.pub; }; c0-8 = { hostNames = [ "c0-8" "c0-8.hpc.local" "10.1.61.108" "10.1.63.108" ]; publicKeyFile = ./pubkeys/c0-8.pub; }; }; environment.systemPackages = [ openssh-shosts ]; security.wrappers = { ssh-keysign = { source = "${openssh-shosts}/libexec/ssh-keysign"; owner = "root"; group = "root"; permissions = "u+rs,g+rx,o+rx"; }; }; }; openssh-shosts = pkgs.openssh.overrideAttrs (attrs: { buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ]; }); myvnc = let myvnc = pkgs.writeScriptBin "myvnc" '' #!${pkgs.runtimeShell} uid=`id -u` port=$((9000+$uid)) shell=`getent passwd $(id -un) | awk -F : '{print $NF}'` # vnc=${pkgs.tigervnc}/bin/vncserver vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver case $1 in -p|--port) shift; port=$1 ;; kill|stop) display=$($vnc -list | sed -n 's/^\(:[0-9]\+\).*/\1/p'| head -1) $vnc -kill $display exit 0 ;; esac ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1 [ $? = 1 ] && $vnc -rfbport $port echo "Xvnc server is running on port $port." exec $shell -i ''; buildCommand = '' mkdir -p $out/bin echo $src > $out/bin/myvnc chmod 755 $out/bin/myvnc ''; in { environment.systemPackages = [ myvnc ]; }; in { options.node = { i40efix = mkEnableOption "Apply fix for i40e driver"; myvnc = mkEnableOption "Enable myvnc script"; }; config = mkMerge [ configuration deployment shosts (mkIf config.node.i40efix i40efix) (mkIf config.node.myvnc myvnc) ]; imports = [ ../../modules ../../nixos ./users.nix ]; }