fix: fix ekman part of botched merge
This commit is contained in:
@@ -13,8 +13,8 @@ let
|
||||
deployment.tags = [ "compute" "c0" ];
|
||||
|
||||
fileSystems = {
|
||||
"/users" = {
|
||||
device = "172.16.239.222:/home";
|
||||
"/frontend" = {
|
||||
device = "10.255.241.100:/home";
|
||||
fsType = "nfs4";
|
||||
options = [
|
||||
"soft"
|
||||
@@ -27,7 +27,7 @@ let
|
||||
|
||||
systemd.automounts = [
|
||||
{
|
||||
where = "/users";
|
||||
where = "/frontend";
|
||||
wantedBy = [ "default.target" ];
|
||||
}
|
||||
];
|
||||
@@ -35,23 +35,20 @@ let
|
||||
|
||||
mkCompute = host:
|
||||
let
|
||||
<<<<<<<< HEAD:rossby/c0/default.nix
|
||||
hw = ./hardware-config.d + "/${host.name}.nix";
|
||||
========
|
||||
hw = ./hardware-configuration.d + "/${host.name}.nix";
|
||||
>>>>>>>> ekman:ekman/c0/default.nix
|
||||
in {
|
||||
"${host.name}" = {
|
||||
cluster = {
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
rdma.enable = true;
|
||||
automount.enable = true;
|
||||
users = true;
|
||||
home = false;
|
||||
opt = true;
|
||||
work = false;
|
||||
work = true;
|
||||
data = false;
|
||||
backup = false;
|
||||
ceph = false;
|
||||
};
|
||||
};
|
||||
@@ -61,9 +58,10 @@ let
|
||||
name = host.name;
|
||||
address = host.address;
|
||||
};
|
||||
os.externalInterface = "enp65s0f0";
|
||||
os.externalInterface = "enp33s0f0np0";
|
||||
hpc.compute = true;
|
||||
# hpc.knem = true;
|
||||
hpc.knem = true;
|
||||
# k8s = { inherit etcdCluster; };
|
||||
};
|
||||
|
||||
deployment.targetHost = host.address;
|
||||
@@ -75,31 +73,31 @@ let
|
||||
networking = {
|
||||
hostName = host.name;
|
||||
useDHCP = false;
|
||||
interfaces. enp65s0f0 = {
|
||||
interfaces.enp33s0f0np0 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = host.address;
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
# ipv4.routes = [ {
|
||||
# address = "10.255.242.0";
|
||||
# prefixLength = 24;
|
||||
# via = "10.255.241.100";
|
||||
# } ];
|
||||
ipv4.routes = [ {
|
||||
address = "10.255.242.0";
|
||||
prefixLength = 24;
|
||||
via = "10.255.241.100";
|
||||
} ];
|
||||
|
||||
};
|
||||
# interfaces."ibp1s0.7666" = {
|
||||
# interfaces."ibp1s0" = {
|
||||
# useDHCP = false;
|
||||
# ipv4.addresses = [ {
|
||||
# address = host.ipoib;
|
||||
# prefixLength = 24;
|
||||
# } ];
|
||||
# };
|
||||
interfaces."ibp1s0" = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = host.ipoib;
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
};
|
||||
imports = [
|
||||
hw
|
||||
../default.nix
|
||||
../cluster.nix
|
||||
../mounts.nix
|
||||
#./kernel.nix
|
||||
];
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/0a0feff4-c3aa-4a67-9880-78c08f2e1db3";
|
||||
{ device = "/dev/disk/by-uuid/7e98da6d-a33d-4845-9123-7e351ad0a39e";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/A52D-B7FA";
|
||||
{ device = "/dev/disk/by-uuid/6ACD-6E5F";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/cd6d1033-a92f-4ab7-8265-00d57466221b";
|
||||
{ device = "/dev/disk/by-uuid/4db39a06-d119-4efb-9111-41c85e622467";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/AEA2-6A41";
|
||||
{ device = "/dev/disk/by-uuid/B8A4-E5C5";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/92a043ce-b2be-4cdd-b6e0-1e8f3e6952fd";
|
||||
{ device = "/dev/disk/by-uuid/4a8e1c95-8bbb-40ed-8865-74bab73e4ee8";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/3038-56D0";
|
||||
{ device = "/dev/disk/by-uuid/7D28-3770";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/5d8be7db-bdb6-47d2-bb9a-5dc62f162c0f";
|
||||
{ device = "/dev/disk/by-uuid/f4579aea-21f4-4d4f-9c65-53f9b540cac4";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/8C14-D452";
|
||||
{ device = "/dev/disk/by-uuid/A20A-720E";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/f0909558-f71a-472e-af38-8ffdb2e9e453";
|
||||
{ device = "/dev/disk/by-uuid/a3fd9989-866c-4d24-9c59-7c2803a28eae";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/72A0-CF21";
|
||||
{ device = "/dev/disk/by-uuid/8B58-FF6B";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/c35b6b6f-29da-4ea0-8fba-067301ce9572";
|
||||
{ device = "/dev/disk/by-uuid/cc8f9ec9-fe2c-4603-8ba1-2df9c8389cbf";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/3FC3-84E2";
|
||||
{ device = "/dev/disk/by-uuid/B26B-7B6E";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/1a765ce5-2156-4427-8594-824473a31205";
|
||||
{ device = "/dev/disk/by-uuid/26d04544-c68a-4e7a-a728-5676f472ebb7";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/4CD7-6D3E";
|
||||
{ device = "/dev/disk/by-uuid/7E82-333A";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/2580dfeb-a28e-4f3d-87a8-e7c7a9fdfeca";
|
||||
{ device = "/dev/disk/by-uuid/6923deaa-ab2e-45c1-bfe6-7fe8be6f1770";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/6BB5-3DC2";
|
||||
{ device = "/dev/disk/by-uuid/C9D4-6AE3";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/032fe2e6-9091-47e8-b121-0123abe4f0fc";
|
||||
{ device = "/dev/disk/by-uuid/d7fdc1a4-39fe-4260-be6c-1857ce972d7e";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/EE11-AD60";
|
||||
{ device = "/dev/disk/by-uuid/0B80-A90E";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/da170cd8-0a6b-470f-96aa-a7bdf3dd8736";
|
||||
{ device = "/dev/disk/by-uuid/aa46e116-30eb-4e49-996b-7ee49fb1914c";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/159D-E4C1";
|
||||
{ device = "/dev/disk/by-uuid/8DF7-E9B8";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/b081d8a1-3cae-4017-afd3-0482d58f3062";
|
||||
{ device = "/dev/disk/by-uuid/886d3243-07c5-4679-9979-cef11a9b5cb1";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/1E63-EA5E";
|
||||
{ device = "/dev/disk/by-uuid/6FA3-B1DE";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/494bceb1-c0d1-419d-a379-8104ff90aa61";
|
||||
{ device = "/dev/disk/by-uuid/db0635c5-e0bf-4d85-9834-f0374f8ffa9c";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/294F-20BD";
|
||||
{ device = "/dev/disk/by-uuid/A3AE-176D";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/be0a7204-1a90-4f06-980a-a315e9c664ca";
|
||||
{ device = "/dev/disk/by-uuid/22b9bf44-cafb-4f06-a23f-9b6f4a748e81";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/4889-6E82";
|
||||
{ device = "/dev/disk/by-uuid/5CBE-8643";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/ffeb8ede-16f1-48bc-a976-1ea9ab982745";
|
||||
{ device = "/dev/disk/by-uuid/de9abab9-3280-4ef2-b467-ea09842b1be6";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/E780-1C40";
|
||||
{ device = "/dev/disk/by-uuid/63E2-0C5A";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,13 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/3d27bbdb-4370-4af2-9747-74d03acfa558";
|
||||
{ device = "/dev/disk/by-uuid/30add79f-20c1-4030-8d72-47a3dc6b533d";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/9E1B-2495";
|
||||
{ device = "/dev/disk/by-uuid/716E-5AC4";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,14 +30,8 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.tailscale0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ with lib;
|
||||
let
|
||||
cfg = config.features.host;
|
||||
|
||||
computeNodes = import ./c0/nodes.nix;
|
||||
computeNodes =
|
||||
import ./c0/nodes.nix ++
|
||||
import ./c1/nodes.nix;
|
||||
|
||||
mkSANs = host: [
|
||||
host.name
|
||||
@@ -51,24 +53,24 @@ let
|
||||
|
||||
time.timeZone = "Europe/Oslo";
|
||||
|
||||
# programs.msmtp = {
|
||||
# enable = false;
|
||||
# accounts = {
|
||||
# default = {
|
||||
# auth = false;
|
||||
# tls = false;
|
||||
# tls_starttls = false;
|
||||
# port = 24;
|
||||
# from = "rossby@oceanbox.io";
|
||||
# host = "smtpgw.itpartner.no";
|
||||
# # user = "utvikling";
|
||||
# # password = "S0m3rp0m@de#21!";
|
||||
# };
|
||||
# };
|
||||
# defaults = {
|
||||
# aliases = "/etc/aliases";
|
||||
# };
|
||||
# };
|
||||
programs.msmtp = {
|
||||
enable = true;
|
||||
accounts = {
|
||||
default = {
|
||||
auth = false;
|
||||
tls = false;
|
||||
tls_starttls = false;
|
||||
port = 24;
|
||||
from = "ekman@oceanbox.io";
|
||||
host = "smtpgw.itpartner.no";
|
||||
# user = "utvikling";
|
||||
# password = "S0m3rp0m@de#21!";
|
||||
};
|
||||
};
|
||||
defaults = {
|
||||
aliases = "/etc/aliases";
|
||||
};
|
||||
};
|
||||
|
||||
environment.etc = {
|
||||
"aliases" = {
|
||||
@@ -99,17 +101,13 @@ let
|
||||
|
||||
networking = {
|
||||
domain = mkDefault "cluster.local";
|
||||
<<<<<<<< HEAD:rossby/default.nix
|
||||
defaultGateway = mkDefault "172.16.239.1";
|
||||
========
|
||||
defaultGateway = mkDefault "10.255.241.1";
|
||||
>>>>>>>> ekman:ekman/default.nix
|
||||
nameservers = mkDefault [ "8.8.8.8" ];
|
||||
search = mkDefault [];
|
||||
extraHosts = import ./hosts.nix;
|
||||
firewall.extraCommands = ''
|
||||
iptables -I INPUT -s 172.16.239.0/24 -j ACCEPT
|
||||
iptables -I INPUT -s 172.16.240.0/24 -j ACCEPT
|
||||
iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
|
||||
iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
|
||||
@@ -146,16 +144,8 @@ let
|
||||
enable = true;
|
||||
client = true;
|
||||
mungeKey = ./munge.key;
|
||||
mungeUid = mkDefault 994; # hack
|
||||
mungeUid = mkDefault 996; # hack
|
||||
# pkey = "0x7666";
|
||||
<<<<<<<< HEAD:rossby/default.nix
|
||||
controlMachine = "rossby-manage";
|
||||
mailDomain = "oceanbox.io";
|
||||
nodeName = [
|
||||
"c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
|
||||
"rossby-login Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
|
||||
"rossby-manage Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
|
||||
========
|
||||
controlMachine = "ekman-manage";
|
||||
mailDomain = "oceanbox.io";
|
||||
nodeName = [
|
||||
@@ -163,14 +153,13 @@ let
|
||||
"c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
|
||||
"ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
|
||||
"ekman-manage Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN"
|
||||
>>>>>>>> ekman:ekman/default.nix
|
||||
];
|
||||
partitionName = [
|
||||
"batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP"
|
||||
"rossby Nodes=rossby-login MaxTime=1:00:00 State=UP"
|
||||
"short Nodes=c0-[1-8] MaxTime=INFINITE State=UP"
|
||||
"long Nodes=c0-[3-8] MaxTime=INFINITE State=UP"
|
||||
"stats Nodes=c0-[7-8] MaxTime=INFINITE State=UP"
|
||||
"batch Nodes=c0-[1-18] Default=YES MaxTime=INFINITE State=UP"
|
||||
"ekman Nodes=ekman MaxTime=1:00:00 State=UP"
|
||||
"short Nodes=c1-[1-8] MaxTime=INFINITE State=UP"
|
||||
"long Nodes=c1-[3-8] MaxTime=INFINITE State=UP"
|
||||
"stats Nodes=c1-[7-8] MaxTime=INFINITE State=UP"
|
||||
];
|
||||
};
|
||||
};
|
||||
@@ -207,26 +196,20 @@ let
|
||||
k8s = {
|
||||
enable = true;
|
||||
node.enable = true;
|
||||
clusterName = "rossby";
|
||||
clusterName = "ekman";
|
||||
initca = ./ca;
|
||||
cidr = "10.10.0.0/16";
|
||||
cidr = "10.100.0.0/16";
|
||||
master = {
|
||||
<<<<<<<< HEAD:rossby/default.nix
|
||||
name = "rossby-manage";
|
||||
address = "172.16.239.221";
|
||||
========
|
||||
name = "ekman-manage";
|
||||
address = "10.255.241.99";
|
||||
>>>>>>>> ekman:ekman/default.nix
|
||||
# extraSANs = [
|
||||
# "rossby-manage.oceanbox.io"
|
||||
# "frontend.oceanbox.io"
|
||||
# ];
|
||||
};
|
||||
ingressNodes = [
|
||||
"rossby-manage.oceanbox.io"
|
||||
"rossby.oceanbox.io"
|
||||
"ekman.oceanbox.io"
|
||||
];
|
||||
fileserver = "rossby-login";
|
||||
fileserver = "fs-work";
|
||||
charts = {
|
||||
acme_email = "acme@oceanbox.io";
|
||||
# grafana_smtp_user = "utvikling";
|
||||
@@ -251,30 +234,20 @@ let
|
||||
uid = 0;
|
||||
gid = 0;
|
||||
text = ''
|
||||
172.16.239.210
|
||||
10.255.241.80
|
||||
10.255.241.90
|
||||
'' + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes;
|
||||
};
|
||||
|
||||
programs.ssh.knownHosts = {
|
||||
<<<<<<<< HEAD:rossby/default.nix
|
||||
rossby-manage = {
|
||||
hostNames = [
|
||||
"manage" "manage.compute.local" "rossby-manage.oceanbox.io" "172.16.239.221" "172.16.240.221"
|
||||
========
|
||||
ekman-manage = {
|
||||
hostNames = [
|
||||
"ekman-manage" "ekman-manage.cluster.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.99"
|
||||
>>>>>>>> ekman:ekman/default.nix
|
||||
];
|
||||
publicKeyFile = ./manage/ssh_host_key.pub;
|
||||
};
|
||||
rossby-login = {
|
||||
ekman = {
|
||||
hostNames = [
|
||||
<<<<<<<< HEAD:rossby/default.nix
|
||||
"rossby" "rpssby-login" "rossby-login.compute.local" "rossby.compute.local" "rossby.oceanbox.io" "172.16.239.222" "172.16.240.222"
|
||||
];
|
||||
publicKeyFile = ./login/ssh_host_key.pub;
|
||||
========
|
||||
"ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100"
|
||||
];
|
||||
publicKeyFile = ./logon/ssh_host_key.pub;
|
||||
@@ -290,29 +263,16 @@ let
|
||||
"fs-backup" "fs-backup.cluster.local" "10.255.241.80" "10.255.243.80"
|
||||
];
|
||||
publicKeyFile = ./fs-backup/ssh_host_key.pub;
|
||||
>>>>>>>> ekman:ekman/default.nix
|
||||
};
|
||||
# fs-work = {
|
||||
# hostNames = [
|
||||
# "fs-work" "fs-work.compute.local" "172.16.239.90" "172.16.240.210"
|
||||
# ];
|
||||
# publicKeyFile = ./fs-work/ssh_host_key.pub;
|
||||
# };
|
||||
} // builtins.foldl' (a: x:
|
||||
let n = toString x.idx;
|
||||
in a // {
|
||||
"${x.name}" = {
|
||||
hostNames = [
|
||||
"${x.name}"
|
||||
<<<<<<<< HEAD:rossby/default.nix
|
||||
"${x.name}.compute.local"
|
||||
"172.16.239.${n}"
|
||||
"172.16.240.${n}"
|
||||
========
|
||||
"${x.name}.cluster.local"
|
||||
"10.255.241.${n}"
|
||||
"10.255.243.${n}"
|
||||
>>>>>>>> ekman:ekman/default.nix
|
||||
];
|
||||
publicKeyFile = x.pubkey;
|
||||
};
|
||||
|
||||
@@ -35,12 +35,13 @@ in {
|
||||
k8sNode = true;
|
||||
slurm = false;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
rdma.enable = true;
|
||||
automount.enable = true;
|
||||
users = false;
|
||||
home = true;
|
||||
opt = false;
|
||||
work = false;
|
||||
data = false;
|
||||
backup = false;
|
||||
ceph = false;
|
||||
};
|
||||
};
|
||||
@@ -164,7 +165,7 @@ in {
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../cluster.nix
|
||||
../mounts.nix
|
||||
];
|
||||
};
|
||||
|
||||
@@ -12,14 +12,14 @@ let
|
||||
allowLocalDeployment = true;
|
||||
targetHost = null;
|
||||
};
|
||||
imports = [ ./manage ];
|
||||
imports = [ ./ekman/manage ];
|
||||
};
|
||||
|
||||
login = import ./login { inherit pkgs; };
|
||||
c0 = import ./c0 { inherit pkgs; };
|
||||
c0x = import ./c0x { inherit pkgs; };
|
||||
c1 = import ./c1 { inherit pkgs; };
|
||||
fs-work = import ./fs-work { inherit pkgs; };
|
||||
fs-backup = import ./fs-backup { inherit pkgs; };
|
||||
ekman-login = import ./ekman/login { inherit pkgs; };
|
||||
c0 = import ./ekman/c0 { inherit pkgs; };
|
||||
c0x = import ./ekman/c0x { inherit pkgs; };
|
||||
c1 = import ./ekman/c1 { inherit pkgs; };
|
||||
fs-work = import ./ekman/fs-work { inherit pkgs; };
|
||||
fs-backup = import ./ekman/fs-backup { inherit pkgs; };
|
||||
in
|
||||
{ inherit ekman-manage; } // login // c0 // c0x // c1 // fs-work // fs-backup
|
||||
{ inherit ekman-manage; } // ekman-login // c0 // c0x // c1 // fs-work // fs-backup
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
let
|
||||
name = "rossby";
|
||||
address = "172.16.239.222";
|
||||
etcdCluster = import ../etcdCluster.nix;
|
||||
name = "ekman";
|
||||
address = "10.255.241.100";
|
||||
in
|
||||
{
|
||||
rossby-login = { config, pkgs, ... }: with pkgs; {
|
||||
deployment.tags = [ "rossby-login" "login" ];
|
||||
ekman = { config, pkgs, ... }: with pkgs; {
|
||||
deployment.tags = [ "ekman" "login" ];
|
||||
deployment.targetHost = address;
|
||||
system.autoUpgrade.enable = lib.mkForce false;
|
||||
|
||||
@@ -21,13 +20,14 @@ in
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = false;
|
||||
rdma.enable = true;
|
||||
automount.enable = true;
|
||||
users = false;
|
||||
home = false;
|
||||
opt = false;
|
||||
work = false;
|
||||
data = false;
|
||||
ceph = false;
|
||||
work = true;
|
||||
data = true;
|
||||
backup = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -40,17 +40,18 @@ in
|
||||
myvnc.enable = false;
|
||||
|
||||
os = {
|
||||
externalInterface = "enp129s0f0";
|
||||
externalInterface = "enp33s0f0np0";
|
||||
nfs.enable = true;
|
||||
nfs.exports = ''
|
||||
/exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
/exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
/exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
hpc = {
|
||||
slurm.server = false;
|
||||
slurm.slurmrestd = false;
|
||||
manager = false;
|
||||
frontend = false;
|
||||
login = true;
|
||||
knem = false;
|
||||
};
|
||||
@@ -58,7 +59,6 @@ in
|
||||
k8s = {
|
||||
master.enable = false;
|
||||
node.enable = true;
|
||||
inherit etcdCluster;
|
||||
};
|
||||
|
||||
desktop.enable = false;
|
||||
@@ -66,7 +66,7 @@ in
|
||||
monitoring = {
|
||||
# server = {
|
||||
# enable = false;
|
||||
# scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
|
||||
# scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
|
||||
# defaultAlertReceiver = {
|
||||
# email_configs = [
|
||||
# { to = "jonas.juselius@oceanbox.io"; }
|
||||
@@ -75,7 +75,7 @@ in
|
||||
# pageAlertReceiver = {
|
||||
# webhook_configs = [
|
||||
# {
|
||||
# url = "https://prometheus-msteams.k2.itpartner.no/rossby";
|
||||
# url = "https://prometheus-msteams.k2.itpartner.no/ekman";
|
||||
# http_config = {
|
||||
# tls_config = { insecure_skip_verify = true; };
|
||||
# };
|
||||
@@ -93,7 +93,28 @@ in
|
||||
infiniband-exporter = {
|
||||
enable = true;
|
||||
nameMap = ''
|
||||
# 0xe8ebd3030024981e "c0-1"
|
||||
0x248a07030029d2fc "frontend"
|
||||
0x248a07030029d104 "ekman"
|
||||
0x5aa2e1fffe1edfca "fs-work"
|
||||
0x1c34da0300787798 "fs-backup"
|
||||
0xe8ebd3030024981e "c0-1"
|
||||
0xe8ebd3030024a21a "c0-2"
|
||||
0xe8ebd30300249a3a "c0-3"
|
||||
0xe8ebd30300248b9e "c0-4"
|
||||
0xe8ebd30300248b86 "c0-5"
|
||||
0xe8ebd3030024998a "c0-6"
|
||||
0xe8ebd30300248b8e "c0-7"
|
||||
0xe8ebd3030024999e "c0-8"
|
||||
0xe8ebd30300248fca "c0-9"
|
||||
0xe8ebd3030024a216 "c0-10"
|
||||
0xe8ebd30300248b96 "c0-11"
|
||||
0xe8ebd30300248b9a "c0-12"
|
||||
0xe8ebd303002495d2 "c0-13"
|
||||
0xe8ebd303002495e2 "c0-14"
|
||||
0xe8ebd30300248f42 "c0-15"
|
||||
0xe8ebd303002495e6 "c0-16"
|
||||
0xe8ebd3030024a2a2 "c0-17"
|
||||
0xe8ebd3030024a2ae "c0-18"
|
||||
'';
|
||||
};
|
||||
slurm-exporter = {
|
||||
@@ -108,22 +129,15 @@ in
|
||||
# '';
|
||||
|
||||
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
|
||||
services.flannel.iface = "enp129s0f0";
|
||||
services.flannel.iface = "enp33s0f3np3";
|
||||
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
hostName = name;
|
||||
<<<<<<<< HEAD:rossby/login/default.nix
|
||||
interfaces.enp129s0f0 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
inherit address;
|
||||
========
|
||||
interfaces.enp33s0f3np3 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = address;
|
||||
>>>>>>>> ekman:ekman/login/default.nix
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
# ipv4.routes = [
|
||||
@@ -134,27 +148,27 @@ in
|
||||
# }
|
||||
# ];
|
||||
};
|
||||
# interfaces."ibp65s0f0" = {
|
||||
# useDHCP = false;
|
||||
# ipv4.addresses = [ {
|
||||
# address = "10.255.243.100";
|
||||
# prefixLength = 24;
|
||||
# } ];
|
||||
# };
|
||||
# interfaces."enp65s0f1np1" = {
|
||||
# useDHCP = false;
|
||||
# ipv4.addresses = [ {
|
||||
# address = "10.255.244.100";
|
||||
# prefixLength = 24;
|
||||
# } ];
|
||||
# };
|
||||
# interfaces.enp33s0f0np0 = {
|
||||
# useDHCP = false;
|
||||
# ipv4.addresses = [ {
|
||||
# address = "10.255.242.2";
|
||||
# prefixLength = 24;
|
||||
# } ];
|
||||
# ipv4.routes = [
|
||||
interfaces."ibp65s0f0" = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = "10.255.243.100";
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
interfaces."enp65s0f1np1" = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = "10.255.244.100";
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
interfaces.enp33s0f0np0 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = "10.255.242.2";
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
ipv4.routes = [
|
||||
# {
|
||||
# address = "10.1.8.0";
|
||||
# prefixLength = 24;
|
||||
@@ -165,13 +179,15 @@ in
|
||||
# prefixLength = 24;
|
||||
# via = "10.255.242.1";
|
||||
# }
|
||||
# ];
|
||||
# };
|
||||
defaultGateway = "172.16.239.1";
|
||||
];
|
||||
};
|
||||
defaultGateway = "10.255.242.1";
|
||||
firewall = {
|
||||
allowedTCPPorts = [ 4443 ];
|
||||
extraCommands = ''
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
||||
# needed for nodeport access on k1 and k2
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.2
|
||||
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
||||
'';
|
||||
};
|
||||
};
|
||||
@@ -197,7 +213,7 @@ in
|
||||
device = "/vol/nfs-provisioner";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/users" = {
|
||||
"/frontend" = {
|
||||
device = "/home";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
@@ -212,7 +228,7 @@ in
|
||||
};
|
||||
|
||||
nix.extraOptions = ''
|
||||
# secret-key-files = /etc/nix/rossby.key
|
||||
secret-key-files = /etc/nix/ekman.key
|
||||
'';
|
||||
|
||||
# services.xserver = {
|
||||
@@ -232,8 +248,8 @@ in
|
||||
smtp_smarthost = "smtpgw.itpartner.no";
|
||||
# smtp_auth_username = "utvikling";
|
||||
# smtp_auth_password = "S0m3rp0m@de#21!";
|
||||
smtp_hello = "rossby.oceanbox.io";
|
||||
smtp_from = "noreplyrossby.oceanbox.io";
|
||||
smtp_hello = "ekman.oceanbox.io";
|
||||
smtp_from = "noreply@ekman.oceanbox.io";
|
||||
};
|
||||
|
||||
# services.nginx = {
|
||||
@@ -266,7 +282,7 @@ in
|
||||
# sif = {
|
||||
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
|
||||
# executor = "shell";
|
||||
# tagList = [ "rossby" "sif" ];
|
||||
# tagList = [ "ekman" "sif" ];
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
@@ -326,13 +342,14 @@ in
|
||||
"--login-server=https://headscale.svc.oceanbox.io"
|
||||
"--accept-dns"
|
||||
"--advertise-exit-node"
|
||||
"--advertise-tags=tag:rossby"
|
||||
"--advertise-routes=10.255.241.241.0/24"
|
||||
"--advertise-tags=tag:ekman"
|
||||
];
|
||||
};
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../cluster.nix
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
];
|
||||
|
||||
@@ -8,20 +8,19 @@
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/e859992d-1627-42cc-a59c-178d596fc220";
|
||||
{ device = "/dev/disk/by-uuid/c4a5ea80-2a87-440d-8431-a695d623da24";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/5281-9FFF";
|
||||
{ device = "/dev/disk/by-uuid/D8A3-A04D";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
@@ -31,12 +30,10 @@
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp129s0f0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp129s0f1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp129s0f2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp129s0f3.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp65s0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
|
||||
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
<<<<<<< HEAD:configuration.nix
|
||||
_:
|
||||
{
|
||||
imports = [ ./rossby/manage ];
|
||||
=======
|
||||
{ pkgs, ...}:
|
||||
let
|
||||
computeNodes =
|
||||
@@ -350,5 +345,5 @@ in {
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
];
|
||||
>>>>>>> ekman:ekman/manage/default.nix
|
||||
}
|
||||
|
||||
|
||||
@@ -3,19 +3,19 @@ with lib;
|
||||
let
|
||||
cfg = config.cluster.mounts;
|
||||
|
||||
subnet =
|
||||
if cfg.rdma.enable then "243"
|
||||
else if cfg.gbe100.enable then "244"
|
||||
else "241";
|
||||
|
||||
options =
|
||||
[ "soft" "defaults" "vers=4.2" ] ++
|
||||
(if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []);
|
||||
|
||||
users =
|
||||
if cfg.users then {
|
||||
<<<<<<<< HEAD:rossby/mounts.nix
|
||||
"/users" = {
|
||||
device = "172.16.239.222:/home";
|
||||
========
|
||||
"/frontend" = {
|
||||
device = "10.255.241.100:/home";
|
||||
>>>>>>>> ekman:ekman/mounts.nix
|
||||
fsType = "nfs4";
|
||||
options = [
|
||||
"soft"
|
||||
@@ -28,7 +28,7 @@ let
|
||||
|
||||
opt =
|
||||
let
|
||||
server = "172.16.239.222";
|
||||
server = "10.255.241.100";
|
||||
in
|
||||
if cfg.opt then {
|
||||
"/opt/bin" = {
|
||||
@@ -54,34 +54,49 @@ let
|
||||
device = "/ceph";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
} else if cfg.backup then {
|
||||
"/data" = {
|
||||
device = "/backup";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
} else {};
|
||||
# device = "10.255.${subnet}.80:/backup";
|
||||
# # device = "10.255.${subnet}.80:/data";
|
||||
# fsType = "nfs4";
|
||||
# inherit options;
|
||||
# };
|
||||
|
||||
work =
|
||||
if cfg.work then {
|
||||
"/work" = {
|
||||
device = "172.16.240.210:/work";
|
||||
device = "10.255.${subnet}.90:/work";
|
||||
fsType = "nfs4";
|
||||
options = options ++ (if cfg.rdma.enable then [ "rdma" ] else []);
|
||||
};
|
||||
} else {};
|
||||
|
||||
backup =
|
||||
if cfg.backup then {
|
||||
"/backup" = {
|
||||
device = "10.255.${subnet}.80:/backup";
|
||||
fsType = "nfs4";
|
||||
options = options ++ [ "ro" ] ++ (if cfg.rdma.enable then [ "rdma" ] else []);
|
||||
};
|
||||
} else {};
|
||||
|
||||
ceph =
|
||||
if cfg.ceph then {
|
||||
"/ceph" = {
|
||||
device = "oceanbox@.data=/";
|
||||
fsType = "ceph";
|
||||
options = [
|
||||
"mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789"
|
||||
"mon_addr=10.255.241.30/10.255.241.31/10.255.241.32:6789"
|
||||
"_netdev"
|
||||
];
|
||||
};
|
||||
} else {};
|
||||
|
||||
<<<<<<<< HEAD:rossby/mounts.nix
|
||||
fileSystems = users // opt // data // work // ceph;
|
||||
========
|
||||
fileSystems = users // opt // data // work // backup // ceph;
|
||||
>>>>>>>> ekman:ekman/mounts.nix
|
||||
|
||||
automount = mountpoint:
|
||||
if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then
|
||||
@@ -106,12 +121,12 @@ let
|
||||
"ceph/ceph.conf" = {
|
||||
text = ''
|
||||
[global]
|
||||
mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789
|
||||
mon_host = 10.255.241.30:6789,10.255.241.31:6789,10.244.241.32:6789
|
||||
log file = /tmp/ceph-$pid.log
|
||||
[client.oceanbox]
|
||||
key = replaceme
|
||||
key = AQDQNgRm6IE7JxAA1glJKsWPIBB/H/GxFYM0vQ==
|
||||
[client.rbd]
|
||||
key = replaceme
|
||||
key = AQCjth9mjR41ABAAvSs6hltidQT6Hu5OKwWu+Q==
|
||||
'';
|
||||
mode = "0660";
|
||||
group = "admin";
|
||||
@@ -128,6 +143,7 @@ in
|
||||
opt = mkEnableOption "Enable /opt";
|
||||
data = mkEnableOption "Enable /data";
|
||||
work = mkEnableOption "Enable /work";
|
||||
backup = mkEnableOption "Enable /backup";
|
||||
ceph = mkEnableOption "Enable /ceph";
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user