{ pkgs ? import {} }: let name = "ekman"; address = "10.255.241.100"; in { ekman = { config, pkgs, ... }: with pkgs; { deployment.tags = [ "ekman" "login" ]; deployment.targetHost = address; system.autoUpgrade.enable = lib.mkForce false; systemd.targets = { sleep.enable = false; suspend.enable = false; hibernate.enable = false; hybrid-sleep.enable = false; }; cluster = { compute = true; k8sNode = true; mounts = { rdma.enable = true; automount.enable = true; users = false; opt = false; work = true; data = true; backup = false; ceph = true; }; }; features = { host = { inherit name; inherit address; }; myvnc.enable = false; os = { externalInterface = "enp33s0f0np0"; nfs.enable = true; nfs.exports = '' /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) ''; }; hpc = { slurm.server = false; slurm.slurmrestd = false; manageNode = false; loginNode = true; knem = false; }; k8s = { master.enable = false; node.enable = true; }; desktop.enable = false; # server.enable = true; monitoring = { # server = { # enable = false; # scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); # defaultAlertReceiver = { # email_configs = [ # { to = "jonas.juselius@oceanbox.io"; } # ]; # }; # pageAlertReceiver = { # webhook_configs = [ # { # url = "https://prometheus-msteams.k2.itpartner.no/ekman"; # http_config = { # tls_config = { insecure_skip_verify = true; }; # }; # } # ]; # }; # }; # webUI.enable = false; # webUI.acmeEmail = "innovasjon@itpartner.no"; # webUI.allow = [ # "10.1.2.0/24" # "172.19.254.0/24" # "172.19.255.0/24" # ]; infiniband-exporter = { enable = true; nameMap = '' 0x248a07030029d2fc "frontend" 0x248a07030029d104 "ekman" 0x5aa2e1fffe1edfca "fs-work" 0x1c34da0300787798 "fs-backup" 0xe8ebd3030024981e "c0-1" 0xe8ebd3030024a21a "c0-2" 0xe8ebd30300249a3a "c0-3" 0xe8ebd30300248b9e "c0-4" 0xe8ebd30300248b86 "c0-5" 0xe8ebd3030024998a "c0-6" 0xe8ebd30300248b8e "c0-7" 0xe8ebd3030024999e "c0-8" 0xe8ebd30300248fca "c0-9" 0xe8ebd3030024a216 "c0-10" 0xe8ebd30300248b96 "c0-11" 0xe8ebd30300248b9a "c0-12" 0xe8ebd303002495d2 "c0-13" 0xe8ebd303002495e2 "c0-14" 0xe8ebd30300248f42 "c0-15" 0xe8ebd303002495e6 "c0-16" 0xe8ebd3030024a2a2 "c0-17" 0xe8ebd3030024a2ae "c0-18" ''; }; slurm-exporter = { enable = true; port = 6080; }; }; }; # services.udev.extraRules = '' # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" # ''; # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6; services.flannel.iface = "enp33s0f3np3"; networking = { useDHCP = false; hostName = name; interfaces.enp33s0f3np3 = { useDHCP = false; ipv4.addresses = [ { address = address; prefixLength = 24; } ]; # ipv4.routes = [ # { # address = "10.255.244.0"; # prefixLength = 24; # via = "10.255.241.99"; # } # ]; }; interfaces."ibp65s0f0" = { useDHCP = false; ipv4.addresses = [ { address = "10.255.243.100"; prefixLength = 24; } ]; }; interfaces."enp65s0f1np1" = { useDHCP = false; ipv4.addresses = [ { address = "10.255.244.100"; prefixLength = 24; } ]; }; interfaces.enp33s0f0np0 = { useDHCP = false; ipv4.addresses = [ { address = "10.255.242.2"; prefixLength = 24; } ]; ipv4.routes = [ # { # address = "10.1.8.0"; # prefixLength = 24; # via = "10.255.242.1"; # } # { # address = "10.1.30.0"; # prefixLength = 24; # via = "10.255.242.1"; # } ]; }; defaultGateway = "10.255.242.1"; firewall = { allowedTCPPorts = [ 6443 ]; extraCommands = '' # needed for nodeport access on k1 and k2 # iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.2 iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE ''; }; }; fileSystems = { "/exports/users" = { device = "/home"; options = [ "bind" ]; }; "/exports/opt/bin" = { device = "/opt/bin"; options = [ "bind" ]; }; "/exports/opt/sif" = { device = "/opt/sif"; options = [ "bind" ]; }; "/exports/nfs-provisioner" = { device = "/vol/nfs-provisioner"; options = [ "bind" ]; }; "/users" = { device = "/home"; options = [ "bind" ]; }; "/vol/local-storage/vol1" = { device = "/vol/vol1"; options = [ "bind" ]; }; "/vol/local-storage/vol2" = { device = "/vol/vol2"; options = [ "bind" ]; }; }; nix.extraOptions = '' secret-key-files = /etc/nix/ekman.key ''; # services.xserver = { # enable = false; # enableCtrlAltBackspace = true; # layout = "us"; # xkbVariant = "altgr-intl"; # xkbOptions = "eurosign:e"; # displayManager = { # gdm.enable = false; # job.logToFile = true; # }; # # desktopManager.xfce.enable = true; # }; services.prometheus.alertmanager.configuration.global = { smtp_smarthost = "smtpgw.itpartner.no"; # smtp_auth_username = "utvikling"; # smtp_auth_password = "S0m3rp0m@de#21!"; smtp_hello = "ekman.oceanbox.io"; smtp_from = "noreply@ekman.oceanbox.io"; }; # services.nginx = { # virtualHosts = { # "ds.matnoc.regnekraft.io" = { # forceSSL = true; # enableACME = true; # serverAliases = []; # locations."/" = { # proxyPass = "http://localhost:9088"; # proxyWebsockets = false; # extraConfig = '' # allow 10.1.2.0/24; # allow 172.19.254.0/24; # allow 172.19.255.0/24; # deny all; # ''; # }; # }; # }; # }; # services.gitlab-runner = { # enable = true; # extraPackages = with pkgs; [ # singularity # ]; # concurrent = 4; # services = { # sif = { # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; # executor = "shell"; # tagList = [ "ekman" "sif" ]; # }; # }; # }; security.sudo.extraConfig = '' %sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity %admin ALL=(admin) NOPASSWD: ALL # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity ''; security.pam = { services.sshd.googleAuthenticator.enable = true; loginLimits = [ { domain = "@users"; item = "rss"; type = "hard"; value = 16000000; } { domain = "@users"; item = "cpu"; type = "hard"; value = 180; } ]; }; system.activationScripts = { home-permissions.text = '' chmod 755 /home/olean chmod 755 /home/frankgaa chmod 755 /home/jonas chmod 755 /home/mrtz chmod 755 /home/avle chmod 755 /home/stig chmod 755 /home/bast chmod 755 /home/simenlk chmod 755 /work/kraken ''; }; # ssh-rsa is deprecated, but putty/winscp users use it services.openssh.extraConfig = '' # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 PubkeyAuthOptions verify-required ''; environment.systemPackages = []; virtualisation.docker.enable = pkgs.lib.mkForce true; services.tailscale = { enable = true; authKeyFile = "/var/lib/secrets/tailscale.key"; useRoutingFeatures = "server"; # for exit-node usage extraUpFlags = [ "--login-server=https://headscale.svc.oceanbox.io" "--accept-dns=false" "--advertise-exit-node" "--advertise-routes=10.255.241.0/24" "--advertise-tags=tag:ekman" ]; }; imports = [ ./hardware-configuration.nix ../default.nix ../mounts.nix ../myvnc.nix ]; }; }