349 lines
9.1 KiB
Nix
349 lines
9.1 KiB
Nix
{ pkgs ? import <nixpkgs> {} }:
|
|
let
|
|
name = "rossby";
|
|
address = "172.16.239.222";
|
|
etcdCluster = import ../etcdCluster.nix;
|
|
in
|
|
{
|
|
rossby-login = { config, pkgs, ... }: with pkgs; {
|
|
deployment.tags = [ "login" "cluster" ];
|
|
deployment.targetHost = address;
|
|
system.autoUpgrade.enable = lib.mkForce false;
|
|
|
|
systemd.targets = {
|
|
sleep.enable = false;
|
|
suspend.enable = false;
|
|
hibernate.enable = false;
|
|
hybrid-sleep.enable = false;
|
|
};
|
|
|
|
cluster = {
|
|
compute = true;
|
|
k8sNode = true;
|
|
mounts = {
|
|
rdma.enable = false;
|
|
automount.enable = true;
|
|
users = true;
|
|
opt = true;
|
|
work = true;
|
|
data = true;
|
|
ceph = true;
|
|
};
|
|
};
|
|
|
|
features = {
|
|
host = {
|
|
inherit name;
|
|
inherit address;
|
|
};
|
|
|
|
myvnc.enable = false;
|
|
|
|
os = {
|
|
externalInterface = "enp65s0np0";
|
|
nfs.enable = true;
|
|
nfs.exports = ''
|
|
/exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
|
'';
|
|
};
|
|
|
|
hpc = {
|
|
slurm.server = false;
|
|
slurm.slurmrestd = false;
|
|
manage = false;
|
|
login = true;
|
|
knem = false;
|
|
};
|
|
|
|
k8s = {
|
|
master.enable = false;
|
|
node.enable = true;
|
|
inherit etcdCluster;
|
|
};
|
|
|
|
desktop.enable = false;
|
|
# server.enable = true;
|
|
monitoring = {
|
|
# server = {
|
|
# enable = false;
|
|
# scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
|
|
# defaultAlertReceiver = {
|
|
# email_configs = [
|
|
# { to = "jonas.juselius@oceanbox.io"; }
|
|
# ];
|
|
# };
|
|
# pageAlertReceiver = {
|
|
# webhook_configs = [
|
|
# {
|
|
# url = "https://prometheus-msteams.k2.itpartner.no/rossby";
|
|
# http_config = {
|
|
# tls_config = { insecure_skip_verify = true; };
|
|
# };
|
|
# }
|
|
# ];
|
|
# };
|
|
# };
|
|
# webUI.enable = false;
|
|
# webUI.acmeEmail = "innovasjon@itpartner.no";
|
|
# webUI.allow = [
|
|
# "10.1.2.0/24"
|
|
# "172.19.254.0/24"
|
|
# "172.19.255.0/24"
|
|
# ];
|
|
infiniband-exporter = {
|
|
enable = true;
|
|
nameMap = ''
|
|
# 0xe8ebd3030024981e "c0-1"
|
|
'';
|
|
};
|
|
slurm-exporter = {
|
|
enable = true;
|
|
port = 6080;
|
|
};
|
|
};
|
|
};
|
|
|
|
# services.udev.extraRules = ''
|
|
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
|
# '';
|
|
|
|
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
|
|
services.flannel.iface = "enp65s0np0";
|
|
|
|
networking = {
|
|
useDHCP = false;
|
|
hostName = name;
|
|
interfaces.enp65s0np0 = {
|
|
useDHCP = false;
|
|
ipv4.addresses = [ {
|
|
inherit address;
|
|
prefixLength = 24;
|
|
} ];
|
|
# ipv4.routes = [
|
|
# {
|
|
# address = "10.255.244.0";
|
|
# prefixLength = 24;
|
|
# via = "10.255.241.99";
|
|
# }
|
|
# ];
|
|
};
|
|
# interfaces."ibp65s0f0" = {
|
|
# useDHCP = false;
|
|
# ipv4.addresses = [ {
|
|
# address = "10.255.243.100";
|
|
# prefixLength = 24;
|
|
# } ];
|
|
# };
|
|
# interfaces."enp65s0f1np1" = {
|
|
# useDHCP = false;
|
|
# ipv4.addresses = [ {
|
|
# address = "10.255.244.100";
|
|
# prefixLength = 24;
|
|
# } ];
|
|
# };
|
|
# interfaces.enp33s0f0np0 = {
|
|
# useDHCP = false;
|
|
# ipv4.addresses = [ {
|
|
# address = "10.255.242.2";
|
|
# prefixLength = 24;
|
|
# } ];
|
|
# ipv4.routes = [
|
|
# {
|
|
# address = "10.1.8.0";
|
|
# prefixLength = 24;
|
|
# via = "10.255.242.1";
|
|
# }
|
|
# {
|
|
# address = "10.1.30.0";
|
|
# prefixLength = 24;
|
|
# via = "10.255.242.1";
|
|
# }
|
|
# ];
|
|
# };
|
|
defaultGateway = "172.16.239.1";
|
|
firewall = {
|
|
allowedTCPPorts = [ 6443 ];
|
|
extraCommands = ''
|
|
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
|
'';
|
|
};
|
|
};
|
|
|
|
fileSystems = {
|
|
"/home" = {
|
|
device = "/ceph/volumes/nfs/home";
|
|
options = [ "bind" ];
|
|
};
|
|
"/vol/local-storage/vol1" = {
|
|
device = "/vol/vol1";
|
|
options = [ "bind" ];
|
|
};
|
|
"/vol/local-storage/vol2" = {
|
|
device = "/vol/vol2";
|
|
options = [ "bind" ];
|
|
};
|
|
# "/exports/home" = {
|
|
# device = "/home";
|
|
# options = [ "bind" ];
|
|
# };
|
|
# "/exports/opt/bin" = {
|
|
# device = "/opt/bin";
|
|
# options = [ "bind" ];
|
|
# };
|
|
# "/exports/opt/sif" = {
|
|
# device = "/opt/sif";
|
|
# options = [ "bind" ];
|
|
# };
|
|
# "/exports/opt/singularity" = {
|
|
# device = "/opt/singularity";
|
|
# options = [ "bind" ];
|
|
# };
|
|
# "/exports/nfs-provisioner" = {
|
|
# device = "/vol/nfs-provisioner";
|
|
# options = [ "bind" ];
|
|
# };
|
|
# "/users" = {
|
|
# device = "/home";
|
|
# options = [ "bind" ];
|
|
# };
|
|
};
|
|
|
|
systemd.automounts = [
|
|
# {
|
|
# wantedBy = [ "multi-user.target" ];
|
|
# automountConfig = {
|
|
# TimeoutIdleSec = "600";
|
|
# };
|
|
# where = "/home";
|
|
# }
|
|
];
|
|
|
|
nix.extraOptions = ''
|
|
# secret-key-files = /etc/nix/rossby.key
|
|
'';
|
|
|
|
# services.xserver = {
|
|
# enable = false;
|
|
# enableCtrlAltBackspace = true;
|
|
# layout = "us";
|
|
# xkbVariant = "altgr-intl";
|
|
# xkbOptions = "eurosign:e";
|
|
# displayManager = {
|
|
# gdm.enable = false;
|
|
# job.logToFile = true;
|
|
# };
|
|
# # desktopManager.xfce.enable = true;
|
|
# };
|
|
|
|
services.prometheus.alertmanager.configuration.global = {
|
|
smtp_smarthost = "smtpgw.itpartner.no";
|
|
# smtp_auth_username = "utvikling";
|
|
# smtp_auth_password = "S0m3rp0m@de#21!";
|
|
smtp_hello = "rossby.oceanbox.io";
|
|
smtp_from = "noreplyrossby.oceanbox.io";
|
|
};
|
|
|
|
# services.nginx = {
|
|
# virtualHosts = {
|
|
# "ds.matnoc.regnekraft.io" = {
|
|
# forceSSL = true;
|
|
# enableACME = true;
|
|
# serverAliases = [];
|
|
# locations."/" = {
|
|
# proxyPass = "http://localhost:9088";
|
|
# proxyWebsockets = false;
|
|
# extraConfig = ''
|
|
# allow 10.1.2.0/24;
|
|
# allow 172.19.254.0/24;
|
|
# allow 172.19.255.0/24;
|
|
# deny all;
|
|
# '';
|
|
# };
|
|
# };
|
|
# };
|
|
# };
|
|
|
|
# services.gitlab-runner = {
|
|
# enable = true;
|
|
# extraPackages = with pkgs; [
|
|
# singularity
|
|
# ];
|
|
# concurrent = 4;
|
|
# services = {
|
|
# sif = {
|
|
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
|
|
# executor = "shell";
|
|
# tagList = [ "rossby" "sif" ];
|
|
# };
|
|
# };
|
|
# };
|
|
|
|
# security.sudo.extraConfig = ''
|
|
# gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
|
# '';
|
|
|
|
security.pam = {
|
|
services.sshd.googleAuthenticator.enable = true;
|
|
loginLimits = [
|
|
{
|
|
domain = "@users";
|
|
item = "rss";
|
|
type = "hard";
|
|
value = 16000000;
|
|
}
|
|
{
|
|
domain = "@users";
|
|
item = "cpu";
|
|
type = "hard";
|
|
value = 180;
|
|
}
|
|
];
|
|
};
|
|
|
|
system.activationScripts = {
|
|
home-permissions.text = ''
|
|
chmod 755 /home/olean
|
|
chmod 755 /home/frankgaa
|
|
chmod 755 /home/jonas
|
|
chmod 755 /home/mrtz
|
|
chmod 755 /home/avle
|
|
chmod 755 /home/stig
|
|
chmod 755 /home/bast
|
|
chmod 755 /home/simenlk
|
|
chmod 755 /work/kraken
|
|
'';
|
|
};
|
|
|
|
|
|
# ssh-rsa is deprecated, but putty/winscp users use it
|
|
services.openssh.extraConfig = ''
|
|
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
|
|
PubkeyAuthOptions verify-required
|
|
'';
|
|
|
|
environment.systemPackages = [];
|
|
|
|
virtualisation.docker.enable = pkgs.lib.mkForce true;
|
|
|
|
services.tailscale = {
|
|
enable = true;
|
|
authKeyFile = "/var/lib/secrets/tailscale.key";
|
|
useRoutingFeatures = "server"; # for exit-node usage
|
|
extraUpFlags = [
|
|
"--login-server=https://headscale.svc.oceanbox.io"
|
|
"--accept-dns"
|
|
"--advertise-exit-node"
|
|
"--advertise-tags=tag:rossby"
|
|
];
|
|
};
|
|
|
|
imports = [
|
|
./hardware-configuration.nix
|
|
../default.nix
|
|
../mounts.nix
|
|
../myvnc.nix
|
|
];
|
|
};
|
|
}
|
|
|