feat: enable IPMI SOL on rossby nodes
This commit is contained in:
Administrator
@@ -1,4 +1,6 @@
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
}:
|
||||
let
|
||||
# Pin the deployment package-set to a specific version of nixpkgs
|
||||
# pkgs = import (builtins.fetchTarball {
|
||||
@@ -10,85 +12,103 @@ let
|
||||
nodes = import ./nodes.nix;
|
||||
|
||||
compute = {
|
||||
deployment.tags = [ "compute" "c0" "cluster" ];
|
||||
deployment.tags = [
|
||||
"compute"
|
||||
"c0"
|
||||
"cluster"
|
||||
];
|
||||
};
|
||||
|
||||
mkCompute = host:
|
||||
let
|
||||
hw = ./hardware-configuration.d + "/${host.name}.nix";
|
||||
in {
|
||||
"${host.name}" = {
|
||||
cluster = {
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
automount.enable = true;
|
||||
users = true;
|
||||
opt = true;
|
||||
work = true;
|
||||
data = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
features = {
|
||||
host = {
|
||||
name = host.name;
|
||||
address = host.address;
|
||||
};
|
||||
os.networkmanager.enable = false;
|
||||
os.externalInterface = host.iface;
|
||||
hpc.computeNode = true;
|
||||
hpc.knem = false;
|
||||
};
|
||||
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
networking = {
|
||||
useNetworkd = true;
|
||||
hostName = host.name;
|
||||
useDHCP = false;
|
||||
};
|
||||
|
||||
# systemd.services.systemd-networkd-wait-online.enable = false;
|
||||
|
||||
systemd.network.wait-online.ignoredInterfaces = [ "ibp1s0" ];
|
||||
systemd.network = {
|
||||
# wait-online.enable = false;
|
||||
networks = {
|
||||
"40-${host.iface}" = {
|
||||
matchConfig.Name = host.iface;
|
||||
address = [ "${host.address}/24" ];
|
||||
networkConfig = { DNSDefaultRoute = true; };
|
||||
routes = [
|
||||
{ Gateway = "172.16.239.1"; }
|
||||
# {
|
||||
# Destination = "10.255.241.0/24";
|
||||
# Gateway = "172.16.239.210";
|
||||
# }
|
||||
];
|
||||
};
|
||||
# "ibp1s0.7666" = {
|
||||
"45-ibp1s0" = {
|
||||
matchConfig.Name = "ibp1s0";
|
||||
address = [ "${host.ipoib}/24" ];
|
||||
mkCompute =
|
||||
host:
|
||||
let
|
||||
hw = ./hardware-configuration.d + "/${host.name}.nix";
|
||||
in
|
||||
{
|
||||
"${host.name}" = {
|
||||
cluster = {
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
automount.enable = true;
|
||||
users = true;
|
||||
opt = true;
|
||||
work = true;
|
||||
data = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
imports = [
|
||||
features = {
|
||||
host = {
|
||||
name = host.name;
|
||||
address = host.address;
|
||||
};
|
||||
os.networkmanager.enable = false;
|
||||
os.externalInterface = host.iface;
|
||||
hpc.computeNode = true;
|
||||
hpc.knem = false;
|
||||
};
|
||||
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
networking = {
|
||||
useNetworkd = true;
|
||||
hostName = host.name;
|
||||
useDHCP = false;
|
||||
};
|
||||
|
||||
# systemd.services.systemd-networkd-wait-online.enable = false;
|
||||
|
||||
systemd.network.wait-online.ignoredInterfaces = [ "ibp1s0" ];
|
||||
systemd.network = {
|
||||
# wait-online.enable = false;
|
||||
networks = {
|
||||
"40-${host.iface}" = {
|
||||
matchConfig.Name = host.iface;
|
||||
address = [ "${host.address}/24" ];
|
||||
networkConfig = {
|
||||
DNSDefaultRoute = true;
|
||||
};
|
||||
routes = [
|
||||
{ Gateway = "172.16.239.1"; }
|
||||
# {
|
||||
# Destination = "10.255.241.0/24";
|
||||
# Gateway = "172.16.239.210";
|
||||
# }
|
||||
];
|
||||
};
|
||||
# "ibp1s0.7666" = {
|
||||
"45-ibp1s0" = {
|
||||
matchConfig.Name = "ibp1s0";
|
||||
address = [ "${host.ipoib}/24" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
boot.kernelParams = [
|
||||
"console=tty1"
|
||||
"console=ttyS1,115200"
|
||||
];
|
||||
systemd.services."serial-getty@ttyS1" = {
|
||||
enable = true;
|
||||
wantedBy = [ "getty.target" ];
|
||||
serviceConfig.Restart = "always";
|
||||
};
|
||||
|
||||
imports = [
|
||||
hw
|
||||
../default.nix
|
||||
../mounts.nix
|
||||
#./kernel.nix
|
||||
];
|
||||
}
|
||||
// compute;
|
||||
};
|
||||
in builtins.foldl' (a: n: a // mkCompute n) {} nodes
|
||||
|
||||
];
|
||||
}
|
||||
// compute;
|
||||
};
|
||||
in
|
||||
builtins.foldl' (a: n: a // mkCompute n) { } nodes
|
||||
|
||||
@@ -25,8 +25,9 @@
|
||||
};
|
||||
|
||||
fileSystems."/work" =
|
||||
{ device = "/dev/disk/by-uuid/baa9d9cc-5ece-45e1-a238-4376700b3ef5";
|
||||
{ device = "/dev/disk/by-uuid/e9bd2849-bff3-4f22-b5b0-92ff4283dcd4";
|
||||
fsType = "xfs";
|
||||
options = [ "nofail" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
@@ -1,220 +1,239 @@
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
}:
|
||||
let
|
||||
name = "rossby";
|
||||
address = "172.16.239.222";
|
||||
etcdCluster = import ../etcdCluster.nix;
|
||||
in
|
||||
{
|
||||
rossby-login = { config, pkgs, ... }: with pkgs; {
|
||||
deployment.tags = [ "login" "cluster" ];
|
||||
deployment.targetHost = address;
|
||||
system.autoUpgrade.enable = lib.mkForce false;
|
||||
rossby-login =
|
||||
{ config, pkgs, ... }:
|
||||
with pkgs;
|
||||
{
|
||||
deployment.tags = [
|
||||
"login"
|
||||
"cluster"
|
||||
];
|
||||
deployment.targetHost = address;
|
||||
system.autoUpgrade.enable = lib.mkForce false;
|
||||
|
||||
systemd.targets = {
|
||||
sleep.enable = false;
|
||||
suspend.enable = false;
|
||||
hibernate.enable = false;
|
||||
hybrid-sleep.enable = false;
|
||||
};
|
||||
|
||||
cluster = {
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
automount.enable = true;
|
||||
users = true;
|
||||
opt = true;
|
||||
work = true;
|
||||
data = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
features = {
|
||||
host = {
|
||||
inherit name;
|
||||
inherit address;
|
||||
systemd.targets = {
|
||||
sleep.enable = false;
|
||||
suspend.enable = false;
|
||||
hibernate.enable = false;
|
||||
hybrid-sleep.enable = false;
|
||||
};
|
||||
|
||||
myvnc.enable = false;
|
||||
|
||||
os = {
|
||||
networkmanager.enable = false;
|
||||
externalInterface = "enp65s0np0";
|
||||
nfs.enable = true;
|
||||
nfs.exports = ''
|
||||
/exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
cluster = {
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
automount.enable = true;
|
||||
users = true;
|
||||
opt = true;
|
||||
work = true;
|
||||
data = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
hpc = {
|
||||
slurm.server = false;
|
||||
slurm.slurmrestd = false;
|
||||
manageNode = false;
|
||||
loginNode = true;
|
||||
knem = false;
|
||||
features = {
|
||||
host = {
|
||||
inherit name;
|
||||
inherit address;
|
||||
};
|
||||
|
||||
myvnc.enable = false;
|
||||
|
||||
os = {
|
||||
networkmanager.enable = false;
|
||||
externalInterface = "enp65s0np0";
|
||||
nfs.enable = true;
|
||||
nfs.exports = ''
|
||||
/exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
hpc = {
|
||||
slurm.server = false;
|
||||
slurm.slurmrestd = false;
|
||||
manageNode = false;
|
||||
loginNode = true;
|
||||
knem = false;
|
||||
};
|
||||
|
||||
k8s = {
|
||||
master.enable = false;
|
||||
node.enable = true;
|
||||
inherit etcdCluster;
|
||||
};
|
||||
|
||||
desktop.enable = false;
|
||||
# server.enable = true;
|
||||
monitoring = {
|
||||
# server = {
|
||||
# enable = false;
|
||||
# scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
|
||||
# defaultAlertReceiver = {
|
||||
# email_configs = [
|
||||
# { to = "jonas.juselius@oceanbox.io"; }
|
||||
# ];
|
||||
# };
|
||||
# };
|
||||
# webUI.enable = false;
|
||||
# webUI.acmeEmail = "noreply@oceanbox.io";
|
||||
# webUI.allow = [
|
||||
# "10.1.2.0/24"
|
||||
# "172.19.254.0/24"
|
||||
# "172.19.255.0/24"
|
||||
# ];
|
||||
};
|
||||
};
|
||||
|
||||
k8s = {
|
||||
master.enable = false;
|
||||
node.enable = true;
|
||||
inherit etcdCluster;
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
|
||||
services.flannel.iface = "enp65s0np0";
|
||||
|
||||
networking = {
|
||||
useNetworkd = true;
|
||||
hostName = name;
|
||||
firewall = {
|
||||
allowedTCPPorts = [ 6443 ];
|
||||
extraCommands = ''
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
desktop.enable = false;
|
||||
# server.enable = true;
|
||||
monitoring = {
|
||||
# server = {
|
||||
# enable = false;
|
||||
# scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
|
||||
# defaultAlertReceiver = {
|
||||
# email_configs = [
|
||||
# { to = "jonas.juselius@oceanbox.io"; }
|
||||
# ];
|
||||
systemd.network = {
|
||||
networks."40-enp65s0np0" = {
|
||||
matchConfig.Name = "enp65s0np0";
|
||||
address = [ "${address}/24" ];
|
||||
networkConfig = {
|
||||
DNSDefaultRoute = true;
|
||||
};
|
||||
routes = [
|
||||
{ Gateway = "172.16.239.1"; }
|
||||
# {
|
||||
# Destination = "10.255.241.0/24";
|
||||
# Gateway = "172.16.239.210";
|
||||
# }
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/home" = {
|
||||
device = "/ceph/volumes/nfs/home";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/vol/local-storage/vol1" = {
|
||||
device = "/vol/vol1";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/vol/local-storage/vol2" = {
|
||||
device = "/vol/vol2";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
};
|
||||
|
||||
nix.extraOptions = ''
|
||||
# secret-key-files = /etc/nix/rossby.key
|
||||
'';
|
||||
|
||||
# services.gitlab-runner = {
|
||||
# enable = true;
|
||||
# extraPackages = with pkgs; [
|
||||
# singularity
|
||||
# ];
|
||||
# concurrent = 4;
|
||||
# services = {
|
||||
# sif = {
|
||||
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
|
||||
# executor = "shell";
|
||||
# tagList = [ "rossby" "sif" ];
|
||||
# };
|
||||
# };
|
||||
# webUI.enable = false;
|
||||
# webUI.acmeEmail = "noreply@oceanbox.io";
|
||||
# webUI.allow = [
|
||||
# "10.1.2.0/24"
|
||||
# "172.19.254.0/24"
|
||||
# "172.19.255.0/24"
|
||||
# ];
|
||||
};
|
||||
};
|
||||
# };
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
security.sudo.extraConfig = ''
|
||||
%sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
||||
%admin ALL=(admin) NOPASSWD: ALL
|
||||
# gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
||||
'';
|
||||
|
||||
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
|
||||
services.flannel.iface = "enp65s0np0";
|
||||
|
||||
networking = {
|
||||
useNetworkd = true;
|
||||
hostName = name;
|
||||
firewall = {
|
||||
allowedTCPPorts = [ 6443 ];
|
||||
extraCommands = ''
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
systemd.network = {
|
||||
networks."40-enp65s0np0" = {
|
||||
matchConfig.Name = "enp65s0np0";
|
||||
address = [ "${address}/24" ];
|
||||
networkConfig = { DNSDefaultRoute = true; };
|
||||
routes = [
|
||||
{ Gateway = "172.16.239.1"; }
|
||||
# {
|
||||
# Destination = "10.255.241.0/24";
|
||||
# Gateway = "172.16.239.210";
|
||||
# }
|
||||
security.pam = {
|
||||
services.sshd.googleAuthenticator.enable = true;
|
||||
loginLimits = [
|
||||
{
|
||||
domain = "@users";
|
||||
item = "rss";
|
||||
type = "hard";
|
||||
value = 16000000;
|
||||
}
|
||||
{
|
||||
domain = "@users";
|
||||
item = "cpu";
|
||||
type = "hard";
|
||||
value = 180;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/home" = {
|
||||
device = "/ceph/volumes/nfs/home";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/vol/local-storage/vol1" = {
|
||||
device = "/vol/vol1";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/vol/local-storage/vol2" = {
|
||||
device = "/vol/vol2";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
};
|
||||
|
||||
nix.extraOptions = ''
|
||||
# secret-key-files = /etc/nix/rossby.key
|
||||
'';
|
||||
|
||||
# services.gitlab-runner = {
|
||||
# enable = true;
|
||||
# extraPackages = with pkgs; [
|
||||
# singularity
|
||||
# ];
|
||||
# concurrent = 4;
|
||||
# services = {
|
||||
# sif = {
|
||||
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
|
||||
# executor = "shell";
|
||||
# tagList = [ "rossby" "sif" ];
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
|
||||
security.sudo.extraConfig = ''
|
||||
%sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
||||
%admin ALL=(admin) NOPASSWD: ALL
|
||||
# gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
|
||||
'';
|
||||
|
||||
security.pam = {
|
||||
services.sshd.googleAuthenticator.enable = true;
|
||||
loginLimits = [
|
||||
{
|
||||
domain = "@users";
|
||||
item = "rss";
|
||||
type = "hard";
|
||||
value = 16000000;
|
||||
}
|
||||
{
|
||||
domain = "@users";
|
||||
item = "cpu";
|
||||
type = "hard";
|
||||
value = 180;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
system.activationScripts = {
|
||||
system.activationScripts = {
|
||||
home-permissions.text = ''
|
||||
chmod 755 /home/olean
|
||||
chmod 755 /home/frankgaa
|
||||
chmod 755 /home/jonas
|
||||
chmod 755 /home/mrtz
|
||||
chmod 755 /home/avle
|
||||
chmod 755 /home/stig
|
||||
chmod 755 /home/bast
|
||||
chmod 755 /home/simenlk
|
||||
chmod 755 /work/kraken
|
||||
chmod 755 /home/olean
|
||||
chmod 755 /home/frankgaa
|
||||
chmod 755 /home/jonas
|
||||
chmod 755 /home/mrtz
|
||||
chmod 755 /home/avle
|
||||
chmod 755 /home/stig
|
||||
chmod 755 /home/bast
|
||||
chmod 755 /home/simenlk
|
||||
chmod 755 /work/kraken
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# ssh-rsa is deprecated, but putty/winscp users use it
|
||||
services.openssh.extraConfig = ''
|
||||
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
|
||||
PubkeyAuthOptions verify-required
|
||||
'';
|
||||
|
||||
# ssh-rsa is deprecated, but putty/winscp users use it
|
||||
services.openssh.extraConfig = ''
|
||||
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
|
||||
PubkeyAuthOptions verify-required
|
||||
'';
|
||||
environment.systemPackages = [ ];
|
||||
|
||||
environment.systemPackages = [];
|
||||
virtualisation.docker.enable = pkgs.lib.mkForce true;
|
||||
|
||||
virtualisation.docker.enable = pkgs.lib.mkForce true;
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
useRoutingFeatures = "client";
|
||||
extraUpFlags = [
|
||||
"--login-server=https://headscale.svc.oceanbox.io"
|
||||
"--accept-dns=true"
|
||||
"--advertise-tags=tag:hpc"
|
||||
];
|
||||
};
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
useRoutingFeatures = "client";
|
||||
extraUpFlags = [
|
||||
"--login-server=https://headscale.svc.oceanbox.io"
|
||||
"--accept-dns=true"
|
||||
"--advertise-tags=tag:hpc"
|
||||
boot.kernelParams = [
|
||||
"console=tty1"
|
||||
"console=ttyS1,115200"
|
||||
];
|
||||
systemd.services."serial-getty@ttyS1" = {
|
||||
enable = true;
|
||||
wantedBy = [ "getty.target" ];
|
||||
serviceConfig.Restart = "always";
|
||||
};
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
];
|
||||
};
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,355 +1,373 @@
|
||||
{ pkgs, ...}:
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
computeNodes =
|
||||
import ../c0/nodes.nix ++
|
||||
[
|
||||
rec {
|
||||
idx = 222;
|
||||
name = "rossby";
|
||||
address = "172.16.239.${toString idx}";
|
||||
ipoib = "10.16.239.${toString idx}";
|
||||
pubkey = ../login/ssh_host_key.pub;
|
||||
}
|
||||
rec {
|
||||
idx = 210;
|
||||
name = "fs-work";
|
||||
address = "172.16.239.${toString idx}";
|
||||
ipoib = "10.16.239.${toString idx}";
|
||||
pubkey = ../fs-work/ssh_host_key.pub;
|
||||
}
|
||||
];
|
||||
computeNodes = import ../c0/nodes.nix ++ [
|
||||
rec {
|
||||
idx = 222;
|
||||
name = "rossby";
|
||||
address = "172.16.239.${toString idx}";
|
||||
ipoib = "10.16.239.${toString idx}";
|
||||
pubkey = ../login/ssh_host_key.pub;
|
||||
}
|
||||
rec {
|
||||
idx = 210;
|
||||
name = "fs-work";
|
||||
address = "172.16.239.${toString idx}";
|
||||
ipoib = "10.16.239.${toString idx}";
|
||||
pubkey = ../fs-work/ssh_host_key.pub;
|
||||
}
|
||||
];
|
||||
etcdCluster = import ../etcdCluster.nix;
|
||||
name = "rossby-manage";
|
||||
address = "172.16.239.221";
|
||||
ipoib = "10.16.239.221";
|
||||
in {
|
||||
systemd.targets = {
|
||||
sleep.enable = false;
|
||||
suspend.enable = false;
|
||||
hibernate.enable = false;
|
||||
hybrid-sleep.enable = false;
|
||||
in
|
||||
{
|
||||
systemd.targets = {
|
||||
sleep.enable = false;
|
||||
suspend.enable = false;
|
||||
hibernate.enable = false;
|
||||
hybrid-sleep.enable = false;
|
||||
};
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
rdma-core
|
||||
hwloc
|
||||
headscale
|
||||
];
|
||||
|
||||
cluster = {
|
||||
k8sNode = true;
|
||||
compute = false;
|
||||
slurm = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
automount.enable = true;
|
||||
users = false;
|
||||
opt = true;
|
||||
work = false;
|
||||
data = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
features = {
|
||||
desktop.enable = false;
|
||||
cachix.enable = false;
|
||||
|
||||
host = {
|
||||
inherit address;
|
||||
inherit name;
|
||||
};
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
myvnc.enable = false;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
rdma-core
|
||||
hwloc
|
||||
headscale
|
||||
];
|
||||
|
||||
cluster = {
|
||||
k8sNode = true;
|
||||
compute = false;
|
||||
slurm = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
automount.enable = true;
|
||||
users = false;
|
||||
opt = true;
|
||||
work = false;
|
||||
data = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
features = {
|
||||
desktop.enable = false;
|
||||
cachix.enable = false;
|
||||
|
||||
host = {
|
||||
inherit address;
|
||||
inherit name;
|
||||
};
|
||||
|
||||
myvnc.enable = false;
|
||||
|
||||
os = {
|
||||
externalInterface = "enp65s0np0";
|
||||
networkmanager.enable = false;
|
||||
nfs.enable = false;
|
||||
nfs.exports = ''
|
||||
/exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
hpc = {
|
||||
slurm.server = true;
|
||||
slurm.slurmrestd = true;
|
||||
slurm.dbdServer = false;
|
||||
manageNode = true;
|
||||
};
|
||||
|
||||
k8s = {
|
||||
master.enable = true;
|
||||
node.enable = true;
|
||||
nodes = computeNodes;
|
||||
inherit etcdCluster;
|
||||
};
|
||||
|
||||
monitoring = {
|
||||
server = {
|
||||
enable = false;
|
||||
scrapeHosts = [
|
||||
"rossby-login"
|
||||
"rossby-manage"
|
||||
"fs-work"
|
||||
] ++ (builtins.map (x: x.name) computeNodes);
|
||||
defaultAlertReceiver = {
|
||||
email_configs = [
|
||||
{ to = "jonas.juselius@oceanbox.io"; }
|
||||
{ to = "moritz.jorg@oceanbox.io"; }
|
||||
];
|
||||
};
|
||||
# NOTE(mrtz): Should be switched to Grafana IRM
|
||||
# pageAlertReceiver = {
|
||||
# webhook_configs = [
|
||||
# {
|
||||
# url = "";
|
||||
# http_config = {
|
||||
# tls_config = { insecure_skip_verify = true; };
|
||||
# };
|
||||
# }
|
||||
# ];
|
||||
# };
|
||||
};
|
||||
webUI.enable = false;
|
||||
webUI.acmeEmail = "acme@oceanbox.io";
|
||||
webUI.allow = [
|
||||
"10.1.2.0/24"
|
||||
"172.19.254.0/24"
|
||||
"172.19.255.0/24"
|
||||
];
|
||||
infiniband-exporter = {
|
||||
enable = true;
|
||||
nameMap = '' # needs fix
|
||||
0x1070fd0300abcc72 "c0-1"
|
||||
0xb8cef603003440ee "c0-2"
|
||||
0x1070fd0300abb6fa "c0-3"
|
||||
0x1070fd0300abc642 "c0-4"
|
||||
0x043f720300dc7876 "c0-5"
|
||||
0x1070fd0300abc636 "c0-6"
|
||||
0xb8cef6030063105c "c0-7"
|
||||
0xb8cef6030037a476 "c0-8"
|
||||
0xb8cef603003443c6 "c0-9"
|
||||
0xb8cef6030049bdd6 "c0-10"
|
||||
0x043f720300dc7a46 "c0-11"
|
||||
0xb8cef6030034410a "c0-12"
|
||||
0xb8cef6030049ba72 "c0-13"
|
||||
0x1070fd0300abca4a "c0-14"
|
||||
0xb8cef60300343056 "c0-15"
|
||||
0x1070fd0300abb356 "c0-16"
|
||||
0xb8cef60300631770 "c0-17"
|
||||
0x1070fd0300abca36 "c0-18"
|
||||
0x1070fd0300abcd0a "c0-19"
|
||||
0x248a070300c06b90 "switch"
|
||||
'';
|
||||
};
|
||||
slurm-exporter = {
|
||||
enable = true;
|
||||
port = 6080;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
programs.singularity.enable = true;
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
|
||||
|
||||
services.flannel.iface = "enp65s0np0";
|
||||
|
||||
networking = {
|
||||
useNetworkd = true;
|
||||
hostName = name;
|
||||
firewall = {
|
||||
allowedTCPPorts = [ 53 6443 4725 6819 ];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
extraCommands = ''
|
||||
# needed for nodeport access on k1 and k2
|
||||
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
|
||||
iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -d 172.16.239.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
|
||||
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j SNAT --to-source 10.255.242.3
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
systemd.network = {
|
||||
networks = {
|
||||
"40-enp65s0np0" = {
|
||||
DHCP = "no";
|
||||
matchConfig.Name = "enp65s0np0";
|
||||
address = [ "${address}/24" ];
|
||||
networkConfig = { DNSDefaultRoute = true; };
|
||||
routes = [
|
||||
{ Gateway = "172.16.239.1"; }
|
||||
# {
|
||||
# Destination = "10.255.241.0/24";
|
||||
# Gateway = "172.16.239.210";
|
||||
# }
|
||||
];
|
||||
};
|
||||
"45-ibp1s0" = {
|
||||
DHCP = "no";
|
||||
matchConfig.Name = "ibp1s0";
|
||||
address = [ "${ipoib}/24" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.resolved = {
|
||||
# DNS=[::1]:53
|
||||
extraConfig = ''
|
||||
DNSStubListener=no
|
||||
os = {
|
||||
externalInterface = "enp65s0np0";
|
||||
networkmanager.enable = false;
|
||||
nfs.enable = false;
|
||||
nfs.exports = ''
|
||||
/exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/exports/public" = {
|
||||
device = "/srv/public";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
hpc = {
|
||||
slurm.server = true;
|
||||
slurm.slurmrestd = true;
|
||||
slurm.dbdServer = false;
|
||||
manageNode = true;
|
||||
};
|
||||
|
||||
nix.extraOptions = ''
|
||||
# secret-key-files = /etc/nix/ekman.key
|
||||
'';
|
||||
k8s = {
|
||||
master.enable = true;
|
||||
node.enable = true;
|
||||
nodes = computeNodes;
|
||||
inherit etcdCluster;
|
||||
};
|
||||
|
||||
services.prometheus.alertmanager.configuration.global = {
|
||||
smtp_smart_host = "oceanbox-io.mail.protection.outlook.com";
|
||||
smtp_tls_config = {
|
||||
ca_file = "/etc/ssl/certs/ca-bundle.crt";
|
||||
cert_file = "/var/lib/secrets/rossby.oceanbox.io.pem";
|
||||
key_file = "/var/lib/secrets/rossby.oceanbox.io-key.pem";
|
||||
monitoring = {
|
||||
server = {
|
||||
enable = false;
|
||||
scrapeHosts = [
|
||||
"rossby-login"
|
||||
"rossby-manage"
|
||||
"fs-work"
|
||||
]
|
||||
++ (builtins.map (x: x.name) computeNodes);
|
||||
defaultAlertReceiver = {
|
||||
email_configs = [
|
||||
{ to = "jonas.juselius@oceanbox.io"; }
|
||||
{ to = "moritz.jorg@oceanbox.io"; }
|
||||
];
|
||||
};
|
||||
smtp_hello = "rossby.oceanbox.io";
|
||||
smtp_from = "noreply@rossby.oceanbox.io";
|
||||
};
|
||||
|
||||
security.pam = {
|
||||
services.sshd.googleAuthenticator.enable = true;
|
||||
loginLimits = [
|
||||
{
|
||||
domain = "@users";
|
||||
item = "rss";
|
||||
type = "hard";
|
||||
value = 16000000;
|
||||
}
|
||||
{
|
||||
domain = "@users";
|
||||
item = "cpu";
|
||||
type = "hard";
|
||||
value = 180;
|
||||
}
|
||||
# NOTE(mrtz): Should be switched to Grafana IRM
|
||||
# pageAlertReceiver = {
|
||||
# webhook_configs = [
|
||||
# {
|
||||
# url = "";
|
||||
# http_config = {
|
||||
# tls_config = { insecure_skip_verify = true; };
|
||||
# };
|
||||
# }
|
||||
# ];
|
||||
# };
|
||||
};
|
||||
webUI.enable = false;
|
||||
webUI.acmeEmail = "acme@oceanbox.io";
|
||||
webUI.allow = [
|
||||
"10.1.2.0/24"
|
||||
"172.19.254.0/24"
|
||||
"172.19.255.0/24"
|
||||
];
|
||||
};
|
||||
|
||||
system.activationScripts = {
|
||||
home-permissions.text = ''
|
||||
chmod 755 /home/olean
|
||||
chmod 755 /home/frankgaa
|
||||
chmod 755 /home/jonas
|
||||
chmod 755 /home/stig
|
||||
chmod 755 /home/bast
|
||||
chmod 755 /home/mrtz
|
||||
chmod 755 /home/avle
|
||||
chmod 755 /home/simenlk
|
||||
chmod 755 /home/ole
|
||||
'';
|
||||
};
|
||||
|
||||
# ssh-rsa is deprecated, but putty/winscp users use it
|
||||
services.openssh.extraConfig = ''
|
||||
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
|
||||
PubkeyAuthOptions verify-required
|
||||
'';
|
||||
|
||||
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
|
||||
|
||||
virtualisation.docker.enable = pkgs.lib.mkForce true;
|
||||
|
||||
security.acme = {
|
||||
acceptTerms = true;
|
||||
defaults.email = "acme@oceanbox.io";
|
||||
certs."rossby.oceanbox.io" = {
|
||||
dnsProvider = "namecheap";
|
||||
environmentFile = /var/lib/secrets/namecheap.key;
|
||||
dnsPropagationCheck = false;
|
||||
postRun = ''
|
||||
cp fullchain.pem /var/lib/secrets/rossby.oceanbox.io.pem
|
||||
cp key.pem /var/lib/secrets/rossby.oceanbox.io-key.pem
|
||||
chmod 644 /var/lib/secrets/rossby.oceanbox.io.pem
|
||||
chmod 640 /var/lib/secrets/rossby.oceanbox.io-key.pem
|
||||
infiniband-exporter = {
|
||||
enable = true;
|
||||
nameMap = ''
|
||||
# needs fix
|
||||
0x1070fd0300abcc72 "c0-1"
|
||||
0xb8cef603003440ee "c0-2"
|
||||
0x1070fd0300abb6fa "c0-3"
|
||||
0x1070fd0300abc642 "c0-4"
|
||||
0x043f720300dc7876 "c0-5"
|
||||
0x1070fd0300abc636 "c0-6"
|
||||
0xb8cef6030063105c "c0-7"
|
||||
0xb8cef6030037a476 "c0-8"
|
||||
0xb8cef603003443c6 "c0-9"
|
||||
0xb8cef6030049bdd6 "c0-10"
|
||||
0x043f720300dc7a46 "c0-11"
|
||||
0xb8cef6030034410a "c0-12"
|
||||
0xb8cef6030049ba72 "c0-13"
|
||||
0x1070fd0300abca4a "c0-14"
|
||||
0xb8cef60300343056 "c0-15"
|
||||
0x1070fd0300abb356 "c0-16"
|
||||
0xb8cef60300631770 "c0-17"
|
||||
0x1070fd0300abca36 "c0-18"
|
||||
0x1070fd0300abcd0a "c0-19"
|
||||
0x248a070300c06b90 "switch"
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
programs.msmtp = {
|
||||
enable = true;
|
||||
accounts = {
|
||||
default = {
|
||||
auth = false;
|
||||
host = "oceanbox-io.mail.protection.outlook.com";
|
||||
port = 25;
|
||||
from = "noreply@oceanbox.io";
|
||||
tls_starttls = true;
|
||||
tls = true;
|
||||
tls_trust_file = "/etc/ssl/certs/ca-bundle.crt";
|
||||
tls_key_file = "/var/lib/secrets/rossby.oceanbox.io-key.pem";
|
||||
tls_cert_file = "/var/lib/secrets/rossby.oceanbox.io.pem";
|
||||
};
|
||||
};
|
||||
defaults = {
|
||||
aliases = "/etc/aliases";
|
||||
slurm-exporter = {
|
||||
enable = true;
|
||||
port = 6080;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
environment.etc = {
|
||||
"aliases" = {
|
||||
text = ''
|
||||
root: jonas.juselius@oceanbox.io, moritz.jorg@oceanbox.io
|
||||
'';
|
||||
mode = "0644";
|
||||
};
|
||||
};
|
||||
programs.singularity.enable = true;
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
useRoutingFeatures = "both"; # for exit-node usage
|
||||
extraUpFlags = [
|
||||
"--login-server=https://headscale.svc.oceanbox.io"
|
||||
"--accept-dns=true"
|
||||
"--advertise-tags=tag:hpc"
|
||||
"--advertise-routes=172.16.239.0/24,172.16.238.0/24"
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
|
||||
|
||||
services.flannel.iface = "enp65s0np0";
|
||||
|
||||
networking = {
|
||||
useNetworkd = true;
|
||||
hostName = name;
|
||||
firewall = {
|
||||
allowedTCPPorts = [
|
||||
53
|
||||
6443
|
||||
4725
|
||||
6819
|
||||
];
|
||||
allowedUDPPorts = [ 53 ];
|
||||
extraCommands = ''
|
||||
# needed for nodeport access on k1 and k2
|
||||
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
|
||||
iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -d 172.16.239.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
|
||||
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j SNAT --to-source 10.255.242.3
|
||||
'';
|
||||
};
|
||||
services.networkd-dispatcher = {
|
||||
enable = true;
|
||||
rules = {
|
||||
"tailscale-router" = {
|
||||
onState = [ "routable" ];
|
||||
script = ''
|
||||
#!${pkgs.runtimeShell}
|
||||
${pkgs.ethtool}/bin/ethtool -K enp65s0np0 rx-udp-gro-forwarding on rx-gro-list off
|
||||
exit 0
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.network = {
|
||||
networks = {
|
||||
"40-enp65s0np0" = {
|
||||
DHCP = "no";
|
||||
matchConfig.Name = "enp65s0np0";
|
||||
address = [ "${address}/24" ];
|
||||
networkConfig = {
|
||||
DNSDefaultRoute = true;
|
||||
};
|
||||
routes = [
|
||||
{ Gateway = "172.16.239.1"; }
|
||||
# {
|
||||
# Destination = "10.255.241.0/24";
|
||||
# Gateway = "172.16.239.210";
|
||||
# }
|
||||
];
|
||||
};
|
||||
"45-ibp1s0" = {
|
||||
DHCP = "no";
|
||||
matchConfig.Name = "ibp1s0";
|
||||
address = [ "${ipoib}/24" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
../../dns.nix
|
||||
services.resolved = {
|
||||
# DNS=[::1]:53
|
||||
extraConfig = ''
|
||||
DNSStubListener=no
|
||||
'';
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/exports/public" = {
|
||||
device = "/srv/public";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
};
|
||||
|
||||
nix.extraOptions = ''
|
||||
# secret-key-files = /etc/nix/ekman.key
|
||||
'';
|
||||
|
||||
services.prometheus.alertmanager.configuration.global = {
|
||||
smtp_smart_host = "oceanbox-io.mail.protection.outlook.com";
|
||||
smtp_tls_config = {
|
||||
ca_file = "/etc/ssl/certs/ca-bundle.crt";
|
||||
cert_file = "/var/lib/secrets/rossby.oceanbox.io.pem";
|
||||
key_file = "/var/lib/secrets/rossby.oceanbox.io-key.pem";
|
||||
};
|
||||
smtp_hello = "rossby.oceanbox.io";
|
||||
smtp_from = "noreply@rossby.oceanbox.io";
|
||||
};
|
||||
|
||||
security.pam = {
|
||||
services.sshd.googleAuthenticator.enable = true;
|
||||
loginLimits = [
|
||||
{
|
||||
domain = "@users";
|
||||
item = "rss";
|
||||
type = "hard";
|
||||
value = 16000000;
|
||||
}
|
||||
{
|
||||
domain = "@users";
|
||||
item = "cpu";
|
||||
type = "hard";
|
||||
value = 180;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
system.activationScripts = {
|
||||
home-permissions.text = ''
|
||||
chmod 755 /home/olean
|
||||
chmod 755 /home/frankgaa
|
||||
chmod 755 /home/jonas
|
||||
chmod 755 /home/stig
|
||||
chmod 755 /home/bast
|
||||
chmod 755 /home/mrtz
|
||||
chmod 755 /home/avle
|
||||
chmod 755 /home/simenlk
|
||||
chmod 755 /home/ole
|
||||
'';
|
||||
};
|
||||
|
||||
# ssh-rsa is deprecated, but putty/winscp users use it
|
||||
services.openssh.extraConfig = ''
|
||||
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
|
||||
PubkeyAuthOptions verify-required
|
||||
'';
|
||||
|
||||
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
|
||||
|
||||
virtualisation.docker.enable = pkgs.lib.mkForce true;
|
||||
|
||||
security.acme = {
|
||||
acceptTerms = true;
|
||||
defaults.email = "acme@oceanbox.io";
|
||||
certs."rossby.oceanbox.io" = {
|
||||
dnsProvider = "namecheap";
|
||||
environmentFile = /var/lib/secrets/namecheap.key;
|
||||
dnsPropagationCheck = false;
|
||||
postRun = ''
|
||||
cp fullchain.pem /var/lib/secrets/rossby.oceanbox.io.pem
|
||||
cp key.pem /var/lib/secrets/rossby.oceanbox.io-key.pem
|
||||
chmod 644 /var/lib/secrets/rossby.oceanbox.io.pem
|
||||
chmod 640 /var/lib/secrets/rossby.oceanbox.io-key.pem
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
programs.msmtp = {
|
||||
enable = true;
|
||||
accounts = {
|
||||
default = {
|
||||
auth = false;
|
||||
host = "oceanbox-io.mail.protection.outlook.com";
|
||||
port = 25;
|
||||
from = "noreply@oceanbox.io";
|
||||
tls_starttls = true;
|
||||
tls = true;
|
||||
tls_trust_file = "/etc/ssl/certs/ca-bundle.crt";
|
||||
tls_key_file = "/var/lib/secrets/rossby.oceanbox.io-key.pem";
|
||||
tls_cert_file = "/var/lib/secrets/rossby.oceanbox.io.pem";
|
||||
};
|
||||
};
|
||||
defaults = {
|
||||
aliases = "/etc/aliases";
|
||||
};
|
||||
};
|
||||
|
||||
environment.etc = {
|
||||
"aliases" = {
|
||||
text = ''
|
||||
root: jonas.juselius@oceanbox.io, moritz.jorg@oceanbox.io
|
||||
'';
|
||||
mode = "0644";
|
||||
};
|
||||
};
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
useRoutingFeatures = "both"; # for exit-node usage
|
||||
extraUpFlags = [
|
||||
"--login-server=https://headscale.svc.oceanbox.io"
|
||||
"--accept-dns=true"
|
||||
"--advertise-tags=tag:hpc"
|
||||
"--advertise-routes=172.16.239.0/24,172.16.238.0/24"
|
||||
];
|
||||
};
|
||||
services.networkd-dispatcher = {
|
||||
enable = true;
|
||||
rules = {
|
||||
"tailscale-router" = {
|
||||
onState = [ "routable" ];
|
||||
script = ''
|
||||
#!${pkgs.runtimeShell}
|
||||
${pkgs.ethtool}/bin/ethtool -K enp65s0np0 rx-udp-gro-forwarding on rx-gro-list off
|
||||
exit 0
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
boot.kernelParams = [
|
||||
"console=tty1"
|
||||
"console=ttyS1,115200"
|
||||
];
|
||||
systemd.services."serial-getty@ttyS1" = {
|
||||
enable = true;
|
||||
wantedBy = [ "getty.target" ];
|
||||
serviceConfig.Restart = "always";
|
||||
};
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
../../dns.nix
|
||||
];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user