oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: add fs-work and etcd cluster

Browse Source
This commit is contained in:
Jonas Juselius
2025-09-13 07:03:17 +02:00
parent 6767eb21e6
commit 3af5ba3fbd
8 changed files with 49 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rossby/c0/default.nix
View File

@@ -46,7 +46,7 @@ let
automount.enable = true;
users = true;
opt = true;
work = false;
work = true;
data = false;
ceph = false;
};

12
rossby/default.nix
View File

@@ -249,12 +249,12 @@ let
];
publicKeyFile = ./login/ssh_host_key.pub;
};
# fs-work = {
# hostNames = [
# "fs-work" "fs-work.compute.local" "172.16.239.90" "172.16.240.210"
# ];
# publicKeyFile = ./fs-work/ssh_host_key.pub;
# };
fs-work = {
hostNames = [
"fs-work" "fs-work.compute.local" "172.16.239.201" "172.16.240.210"
];
publicKeyFile = ./fs-work/ssh_host_key.pub;
};
} // builtins.foldl' (a: x:
let n = toString x.idx;
in a // {

8
rossby/etcdCluster.nix
View File

@@ -1,11 +1,11 @@
{
enable = true;
existing = false;
existing = true;
nodes =
{
# manage = "172.16.239.221";
# rossby = "172.16.239.222";
# fs-work = "172.16.239.210";
rossby-manage = "172.16.239.221";
rossby = "172.16.239.222";
fs-work = "172.16.239.210";
};
}

52
rossby/fs-work/default.nix
View File

@@ -8,7 +8,7 @@ let
# pkgs = import <nixpkgs> {};
etcdCluster = import ../etcdCluster.nix;
name = "fs-work";
address = "10.255.241.90";
address = "172.16.239.210";
in {
fs-work = { config, pkgs, ... }: with pkgs; {
deployment.tags = [ "fs" "fs-work" "cluster" ];
@@ -54,12 +54,10 @@ in {
};
os = {
externalInterface = "enp33s0f3np3";
externalInterface = "enp129s0f0";
nfs.enable = true;
nfs.exports = ''
/exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
@@ -101,7 +99,7 @@ in {
networking = {
hostName = name;
interfaces.enp65s0f0np0 = {
interfaces.enp129s0f0 = {
useDHCP = false;
ipv4.addresses = [
{
@@ -109,40 +107,20 @@ in {
prefixLength = 24;
}
];
ipv4.routes = [
{
address = "10.255.242.0";
prefixLength = 24;
via = "10.255.241.100";
}
];
};
interfaces.enp1s0f1np1 = {
useDHCP = false;
ipv4.addresses = [
{
address = "10.255.244.90";
prefixLength = 24;
}
];
};
# interfaces."ibp65s0.7666" = {
# useDHCP = false;
# };
interfaces.ibp1s0f0 = {
useDHCP = false;
ipv4.addresses = [
{
address = "10.255.243.90";
prefixLength = 24;
}
];
# ipv4.routes = [
# {
# address = "172.16.239.0";
# prefixLength = 24;
# via = "172.16.239.1";
# }
# ];
};
defaultGateway = "172.16.239.1";
firewall = {
allowedTCPPorts = [];
allowedUDPPorts = [];
extraCommands = ''
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j MASQUERADE
'';
};
};
@@ -154,10 +132,6 @@ in {
device = "/work";
options = [ "bind" ];
};
"/exports/opt" = {
device = "/opt";
options = [ "bind" ];
};
};
programs.singularity.enable = true;

26
rossby/fs-work/hardware-configuration.nix
View File

@@ -8,25 +8,20 @@
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "smartpqi" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
boot.initrd.kernelModules = [ "dm-snapshot" ];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/21d2c2c7-4968-432d-a5c4-08116147b816";
{ device = "/dev/disk/by-uuid/45e5f9e7-7263-443c-9cb1-9143e7dee1d9";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/49B0-2597";
{ device = "/dev/disk/by-uuid/5621-2F9E";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
fileSystems."/work" =
{ device = "/dev/mapper/data-work";
fsType = "xfs";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
@@ -36,11 +31,12 @@
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0f0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0f1np1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1np1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp6s0f4u1u6.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f3.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp66s0np0.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;

2
rossby/fs-work/ssh_host_key.pub
View File

@@ -1 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxcJ+G4e6TEy0CZ66V6rRvODZ3QBgG5uIPElZr0lAFEe2FLPtiR57puXTV4tFhkY/TsVo0BE5/qDcBUzdRJ0ihuVVruvEUJDr+wDby1+V823xvPYT0IqPNvBBcR/FKIUTWFIhDHCntQ3MWvcp+bXX4ZH02QGmnSNNjyH6krOofuNz0/H8UxpLn/U40ruQx5SyT+uc7t0qryEUAwdvkQ6dGqyeCd4szSyQ1RfaTcd3qqXltxfKx5EbRAxD6pLoiKzYBGzWwKOLczNJGsLoMQ53KhEUyAdEOEx79qxoR4PU8P5q7osZD9NuloHgVT2P1hiNcsNmFzze3WW96PyK9L6Y1NE2rfcrhHIHG8tK31YXBlFQWr/VuPHpvqt21z0r2tPJwM5ui1zgZ1tsEWuKURItqr5Xsxk3D3woY5hT4rtDkV0TLe8/PUTGkGtw15UbQ/BnsFctWe5JnTtZ+90Tjxl4Ty/h0AYilVff6I3tfEln9MBnYig/cOUX0lmo2lnYpUedHYq5LXaga15OW0uonMd8Wl0nSGbqpJ0Gmu+gONJU9AbGchgxxmo0LNbdQ4jANq11LhcK2UZpyPKA7zV0pcgo1WSQyewfdFYIOJzGJPbDhOai/0OAXJYPJa/QJvp08ULpGrUUYXtjmVLw0DCYihm4af84Gr5Fu7hqNO1dplbJh8w== root@nixos
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKwpzPg00HNAwP4E/yzw7nTvSeQVUYn5aQhm+J/m0r8k root@nixos

4
rossby/login/default.nix
View File

@@ -25,7 +25,7 @@ in
automount.enable = true;
users = false;
opt = false;
work = false;
work = true;
data = false;
ceph = false;
};
@@ -162,7 +162,7 @@ in
# };
defaultGateway = "172.16.239.1";
firewall = {
allowedTCPPorts = [ 4443 ];
allowedTCPPorts = [ 6443 ];
extraCommands = ''
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';

22
rossby/manage/default.nix
View File

@@ -10,13 +10,13 @@ let
ipoib = "172.16.240.${toString idx}";
pubkey = ../login/ssh_host_key.pub;
}
# rec {
# idx = 210;
# name = "fs-work";
# address = "172.16.239.${toString idx}";
# ipoib = "172.16.240.${toString idx}";
# pubkey = ../fs-work/ssh_host_key.pub;
# }
rec {
idx = 210;
name = "fs-work";
address = "172.16.239.${toString idx}";
ipoib = "172.16.240.${toString idx}";
pubkey = ../fs-work/ssh_host_key.pub;
}
];
etcdCluster = import ../etcdCluster.nix;
name = "rossby-manage";
@@ -47,9 +47,9 @@ in {
mounts = {
rdma.enable = false;
automount.enable = true;
users = false;
opt = false;
work = false;
users = true;
opt = true;
work = true;
data = false;
ceph = false;
};
@@ -181,7 +181,7 @@ in {
# };
defaultGateway = "172.16.239.1";
firewall = {
allowedTCPPorts = [ 4443 4725 ];
allowedTCPPorts = [ 6443 4725 ];
extraCommands = ''
# needed for nodeport access on k1 and k2
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3