diff --git a/ekman/c0/default.nix b/ekman/c0/default.nix index e5e0519..69b3b0d 100644 --- a/ekman/c0/default.nix +++ b/ekman/c0/default.nix @@ -13,8 +13,8 @@ let deployment.tags = [ "compute" "c0" ]; fileSystems = { - "/users" = { - device = "172.16.239.222:/home"; + "/frontend" = { + device = "10.255.241.100:/home"; fsType = "nfs4"; options = [ "soft" @@ -27,7 +27,7 @@ let systemd.automounts = [ { - where = "/users"; + where = "/frontend"; wantedBy = [ "default.target" ]; } ]; @@ -35,23 +35,20 @@ let mkCompute = host: let -<<<<<<<< HEAD:rossby/c0/default.nix - hw = ./hardware-config.d + "/${host.name}.nix"; -======== hw = ./hardware-configuration.d + "/${host.name}.nix"; ->>>>>>>> ekman:ekman/c0/default.nix in { "${host.name}" = { cluster = { compute = true; k8sNode = true; mounts = { - rdma.enable = false; + rdma.enable = true; automount.enable = true; - users = true; + home = false; opt = true; - work = false; + work = true; data = false; + backup = false; ceph = false; }; }; @@ -61,9 +58,10 @@ let name = host.name; address = host.address; }; - os.externalInterface = "enp65s0f0"; + os.externalInterface = "enp33s0f0np0"; hpc.compute = true; - # hpc.knem = true; + hpc.knem = true; + # k8s = { inherit etcdCluster; }; }; deployment.targetHost = host.address; @@ -75,31 +73,31 @@ let networking = { hostName = host.name; useDHCP = false; - interfaces. enp65s0f0 = { + interfaces.enp33s0f0np0 = { useDHCP = false; ipv4.addresses = [ { address = host.address; prefixLength = 24; } ]; - # ipv4.routes = [ { - # address = "10.255.242.0"; - # prefixLength = 24; - # via = "10.255.241.100"; - # } ]; + ipv4.routes = [ { + address = "10.255.242.0"; + prefixLength = 24; + via = "10.255.241.100"; + } ]; }; # interfaces."ibp1s0.7666" = { - # interfaces."ibp1s0" = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = host.ipoib; - # prefixLength = 24; - # } ]; - # }; + interfaces."ibp1s0" = { + useDHCP = false; + ipv4.addresses = [ { + address = host.ipoib; + prefixLength = 24; + } ]; + }; }; imports = [ hw - ../default.nix + ../cluster.nix ../mounts.nix #./kernel.nix ]; diff --git a/ekman/c0/hardware-configuration.d/c0-1.nix b/ekman/c0/hardware-configuration.d/c0-1.nix index 185e1ef..62bba1a 100644 --- a/ekman/c0/hardware-configuration.d/c0-1.nix +++ b/ekman/c0/hardware-configuration.d/c0-1.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/0a0feff4-c3aa-4a67-9880-78c08f2e1db3"; + { device = "/dev/disk/by-uuid/7e98da6d-a33d-4845-9123-7e351ad0a39e"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/A52D-B7FA"; + { device = "/dev/disk/by-uuid/6ACD-6E5F"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-10.nix b/ekman/c0/hardware-configuration.d/c0-10.nix index 88ddfc7..3357395 100644 --- a/ekman/c0/hardware-configuration.d/c0-10.nix +++ b/ekman/c0/hardware-configuration.d/c0-10.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/cd6d1033-a92f-4ab7-8265-00d57466221b"; + { device = "/dev/disk/by-uuid/4db39a06-d119-4efb-9111-41c85e622467"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/AEA2-6A41"; + { device = "/dev/disk/by-uuid/B8A4-E5C5"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-11.nix b/ekman/c0/hardware-configuration.d/c0-11.nix index 23a463a..62df049 100644 --- a/ekman/c0/hardware-configuration.d/c0-11.nix +++ b/ekman/c0/hardware-configuration.d/c0-11.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/92a043ce-b2be-4cdd-b6e0-1e8f3e6952fd"; + { device = "/dev/disk/by-uuid/4a8e1c95-8bbb-40ed-8865-74bab73e4ee8"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/3038-56D0"; + { device = "/dev/disk/by-uuid/7D28-3770"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-12.nix b/ekman/c0/hardware-configuration.d/c0-12.nix index d6e6e1b..6c8d140 100644 --- a/ekman/c0/hardware-configuration.d/c0-12.nix +++ b/ekman/c0/hardware-configuration.d/c0-12.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/5d8be7db-bdb6-47d2-bb9a-5dc62f162c0f"; + { device = "/dev/disk/by-uuid/f4579aea-21f4-4d4f-9c65-53f9b540cac4"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/8C14-D452"; + { device = "/dev/disk/by-uuid/A20A-720E"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-13.nix b/ekman/c0/hardware-configuration.d/c0-13.nix index 19898f1..ffd8eaa 100644 --- a/ekman/c0/hardware-configuration.d/c0-13.nix +++ b/ekman/c0/hardware-configuration.d/c0-13.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/f0909558-f71a-472e-af38-8ffdb2e9e453"; + { device = "/dev/disk/by-uuid/a3fd9989-866c-4d24-9c59-7c2803a28eae"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/72A0-CF21"; + { device = "/dev/disk/by-uuid/8B58-FF6B"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-14.nix b/ekman/c0/hardware-configuration.d/c0-14.nix index 4527c39..91d3467 100644 --- a/ekman/c0/hardware-configuration.d/c0-14.nix +++ b/ekman/c0/hardware-configuration.d/c0-14.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/c35b6b6f-29da-4ea0-8fba-067301ce9572"; + { device = "/dev/disk/by-uuid/cc8f9ec9-fe2c-4603-8ba1-2df9c8389cbf"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/3FC3-84E2"; + { device = "/dev/disk/by-uuid/B26B-7B6E"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-15.nix b/ekman/c0/hardware-configuration.d/c0-15.nix index 693bf56..0830e27 100644 --- a/ekman/c0/hardware-configuration.d/c0-15.nix +++ b/ekman/c0/hardware-configuration.d/c0-15.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/1a765ce5-2156-4427-8594-824473a31205"; + { device = "/dev/disk/by-uuid/26d04544-c68a-4e7a-a728-5676f472ebb7"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/4CD7-6D3E"; + { device = "/dev/disk/by-uuid/7E82-333A"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-16.nix b/ekman/c0/hardware-configuration.d/c0-16.nix index 7323c07..a7a645d 100644 --- a/ekman/c0/hardware-configuration.d/c0-16.nix +++ b/ekman/c0/hardware-configuration.d/c0-16.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/2580dfeb-a28e-4f3d-87a8-e7c7a9fdfeca"; + { device = "/dev/disk/by-uuid/6923deaa-ab2e-45c1-bfe6-7fe8be6f1770"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/6BB5-3DC2"; + { device = "/dev/disk/by-uuid/C9D4-6AE3"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-2.nix b/ekman/c0/hardware-configuration.d/c0-2.nix index 270ddb7..aafd1e4 100644 --- a/ekman/c0/hardware-configuration.d/c0-2.nix +++ b/ekman/c0/hardware-configuration.d/c0-2.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/032fe2e6-9091-47e8-b121-0123abe4f0fc"; + { device = "/dev/disk/by-uuid/d7fdc1a4-39fe-4260-be6c-1857ce972d7e"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/EE11-AD60"; + { device = "/dev/disk/by-uuid/0B80-A90E"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-3.nix b/ekman/c0/hardware-configuration.d/c0-3.nix index eaf215b..72452cc 100644 --- a/ekman/c0/hardware-configuration.d/c0-3.nix +++ b/ekman/c0/hardware-configuration.d/c0-3.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/da170cd8-0a6b-470f-96aa-a7bdf3dd8736"; + { device = "/dev/disk/by-uuid/aa46e116-30eb-4e49-996b-7ee49fb1914c"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/159D-E4C1"; + { device = "/dev/disk/by-uuid/8DF7-E9B8"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-4.nix b/ekman/c0/hardware-configuration.d/c0-4.nix index 3ec10fa..7e29d0e 100644 --- a/ekman/c0/hardware-configuration.d/c0-4.nix +++ b/ekman/c0/hardware-configuration.d/c0-4.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/b081d8a1-3cae-4017-afd3-0482d58f3062"; + { device = "/dev/disk/by-uuid/886d3243-07c5-4679-9979-cef11a9b5cb1"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/1E63-EA5E"; + { device = "/dev/disk/by-uuid/6FA3-B1DE"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-5.nix b/ekman/c0/hardware-configuration.d/c0-5.nix index 0cae6d1..e63952d 100644 --- a/ekman/c0/hardware-configuration.d/c0-5.nix +++ b/ekman/c0/hardware-configuration.d/c0-5.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/494bceb1-c0d1-419d-a379-8104ff90aa61"; + { device = "/dev/disk/by-uuid/db0635c5-e0bf-4d85-9834-f0374f8ffa9c"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/294F-20BD"; + { device = "/dev/disk/by-uuid/A3AE-176D"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-6.nix b/ekman/c0/hardware-configuration.d/c0-6.nix index 4df363a..d541207 100644 --- a/ekman/c0/hardware-configuration.d/c0-6.nix +++ b/ekman/c0/hardware-configuration.d/c0-6.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/be0a7204-1a90-4f06-980a-a315e9c664ca"; + { device = "/dev/disk/by-uuid/22b9bf44-cafb-4f06-a23f-9b6f4a748e81"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/4889-6E82"; + { device = "/dev/disk/by-uuid/5CBE-8643"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-8.nix b/ekman/c0/hardware-configuration.d/c0-8.nix index 310c276..2fa085c 100644 --- a/ekman/c0/hardware-configuration.d/c0-8.nix +++ b/ekman/c0/hardware-configuration.d/c0-8.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/ffeb8ede-16f1-48bc-a976-1ea9ab982745"; + { device = "/dev/disk/by-uuid/de9abab9-3280-4ef2-b467-ea09842b1be6"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/E780-1C40"; + { device = "/dev/disk/by-uuid/63E2-0C5A"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,13 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/c0/hardware-configuration.d/c0-9.nix b/ekman/c0/hardware-configuration.d/c0-9.nix index c91b78f..8eb54b3 100644 --- a/ekman/c0/hardware-configuration.d/c0-9.nix +++ b/ekman/c0/hardware-configuration.d/c0-9.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/3d27bbdb-4370-4af2-9747-74d03acfa558"; + { device = "/dev/disk/by-uuid/30add79f-20c1-4030-8d72-47a3dc6b533d"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/9E1B-2495"; + { device = "/dev/disk/by-uuid/716E-5AC4"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,14 +30,8 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; - # networking.interfaces.tailscale0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/default.nix b/ekman/default.nix index a322082..bf663c7 100644 --- a/ekman/default.nix +++ b/ekman/default.nix @@ -3,7 +3,9 @@ with lib; let cfg = config.features.host; - computeNodes = import ./c0/nodes.nix; + computeNodes = + import ./c0/nodes.nix ++ + import ./c1/nodes.nix; mkSANs = host: [ host.name @@ -51,24 +53,24 @@ let time.timeZone = "Europe/Oslo"; -# programs.msmtp = { -# enable = false; -# accounts = { -# default = { -# auth = false; -# tls = false; -# tls_starttls = false; -# port = 24; -# from = "rossby@oceanbox.io"; -# host = "smtpgw.itpartner.no"; -# # user = "utvikling"; -# # password = "S0m3rp0m@de#21!"; -# }; -# }; -# defaults = { -# aliases = "/etc/aliases"; -# }; -# }; + programs.msmtp = { + enable = true; + accounts = { + default = { + auth = false; + tls = false; + tls_starttls = false; + port = 24; + from = "ekman@oceanbox.io"; + host = "smtpgw.itpartner.no"; + # user = "utvikling"; + # password = "S0m3rp0m@de#21!"; + }; + }; + defaults = { + aliases = "/etc/aliases"; + }; + }; environment.etc = { "aliases" = { @@ -99,17 +101,13 @@ let networking = { domain = mkDefault "cluster.local"; -<<<<<<<< HEAD:rossby/default.nix - defaultGateway = mkDefault "172.16.239.1"; -======== defaultGateway = mkDefault "10.255.241.1"; ->>>>>>>> ekman:ekman/default.nix nameservers = mkDefault [ "8.8.8.8" ]; search = mkDefault []; extraHosts = import ./hosts.nix; firewall.extraCommands = '' - iptables -I INPUT -s 172.16.239.0/24 -j ACCEPT - iptables -I INPUT -s 172.16.240.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT ''; }; @@ -146,16 +144,8 @@ let enable = true; client = true; mungeKey = ./munge.key; - mungeUid = mkDefault 994; # hack + mungeUid = mkDefault 996; # hack # pkey = "0x7666"; -<<<<<<<< HEAD:rossby/default.nix - controlMachine = "rossby-manage"; - mailDomain = "oceanbox.io"; - nodeName = [ - "c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" - "rossby-login Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" - "rossby-manage Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" -======== controlMachine = "ekman-manage"; mailDomain = "oceanbox.io"; nodeName = [ @@ -163,14 +153,13 @@ let "c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" "ekman-manage Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN" ->>>>>>>> ekman:ekman/default.nix ]; partitionName = [ - "batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP" - "rossby Nodes=rossby-login MaxTime=1:00:00 State=UP" - "short Nodes=c0-[1-8] MaxTime=INFINITE State=UP" - "long Nodes=c0-[3-8] MaxTime=INFINITE State=UP" - "stats Nodes=c0-[7-8] MaxTime=INFINITE State=UP" + "batch Nodes=c0-[1-18] Default=YES MaxTime=INFINITE State=UP" + "ekman Nodes=ekman MaxTime=1:00:00 State=UP" + "short Nodes=c1-[1-8] MaxTime=INFINITE State=UP" + "long Nodes=c1-[3-8] MaxTime=INFINITE State=UP" + "stats Nodes=c1-[7-8] MaxTime=INFINITE State=UP" ]; }; }; @@ -207,26 +196,20 @@ let k8s = { enable = true; node.enable = true; - clusterName = "rossby"; + clusterName = "ekman"; initca = ./ca; - cidr = "10.10.0.0/16"; + cidr = "10.100.0.0/16"; master = { -<<<<<<<< HEAD:rossby/default.nix - name = "rossby-manage"; - address = "172.16.239.221"; -======== name = "ekman-manage"; address = "10.255.241.99"; ->>>>>>>> ekman:ekman/default.nix # extraSANs = [ - # "rossby-manage.oceanbox.io" + # "frontend.oceanbox.io" # ]; }; ingressNodes = [ - "rossby-manage.oceanbox.io" - "rossby.oceanbox.io" + "ekman.oceanbox.io" ]; - fileserver = "rossby-login"; + fileserver = "fs-work"; charts = { acme_email = "acme@oceanbox.io"; # grafana_smtp_user = "utvikling"; @@ -251,30 +234,20 @@ let uid = 0; gid = 0; text = '' - 172.16.239.210 + 10.255.241.80 + 10.255.241.90 '' + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes; }; programs.ssh.knownHosts = { -<<<<<<<< HEAD:rossby/default.nix - rossby-manage = { - hostNames = [ - "manage" "manage.compute.local" "rossby-manage.oceanbox.io" "172.16.239.221" "172.16.240.221" -======== ekman-manage = { hostNames = [ "ekman-manage" "ekman-manage.cluster.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.99" ->>>>>>>> ekman:ekman/default.nix ]; publicKeyFile = ./manage/ssh_host_key.pub; }; - rossby-login = { + ekman = { hostNames = [ -<<<<<<<< HEAD:rossby/default.nix - "rossby" "rpssby-login" "rossby-login.compute.local" "rossby.compute.local" "rossby.oceanbox.io" "172.16.239.222" "172.16.240.222" - ]; - publicKeyFile = ./login/ssh_host_key.pub; -======== "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100" ]; publicKeyFile = ./logon/ssh_host_key.pub; @@ -290,29 +263,16 @@ let "fs-backup" "fs-backup.cluster.local" "10.255.241.80" "10.255.243.80" ]; publicKeyFile = ./fs-backup/ssh_host_key.pub; ->>>>>>>> ekman:ekman/default.nix }; - # fs-work = { - # hostNames = [ - # "fs-work" "fs-work.compute.local" "172.16.239.90" "172.16.240.210" - # ]; - # publicKeyFile = ./fs-work/ssh_host_key.pub; - # }; } // builtins.foldl' (a: x: let n = toString x.idx; in a // { "${x.name}" = { hostNames = [ "${x.name}" -<<<<<<<< HEAD:rossby/default.nix - "${x.name}.compute.local" - "172.16.239.${n}" - "172.16.240.${n}" -======== "${x.name}.cluster.local" "10.255.241.${n}" "10.255.243.${n}" ->>>>>>>> ekman:ekman/default.nix ]; publicKeyFile = x.pubkey; }; diff --git a/ekman/fs-work/default.nix b/ekman/fs-work/default.nix index 42ec4ca..5db73d7 100644 --- a/ekman/fs-work/default.nix +++ b/ekman/fs-work/default.nix @@ -35,12 +35,13 @@ in { k8sNode = true; slurm = false; mounts = { - rdma.enable = false; + rdma.enable = true; automount.enable = true; - users = false; + home = true; opt = false; work = false; data = false; + backup = false; ceph = false; }; }; @@ -164,7 +165,7 @@ in { imports = [ ./hardware-configuration.nix - ../default.nix + ../cluster.nix ../mounts.nix ]; }; diff --git a/ekman/hive.nix b/ekman/hive.nix index e674d76..4576084 100644 --- a/ekman/hive.nix +++ b/ekman/hive.nix @@ -12,14 +12,14 @@ let allowLocalDeployment = true; targetHost = null; }; - imports = [ ./manage ]; + imports = [ ./ekman/manage ]; }; - login = import ./login { inherit pkgs; }; - c0 = import ./c0 { inherit pkgs; }; - c0x = import ./c0x { inherit pkgs; }; - c1 = import ./c1 { inherit pkgs; }; - fs-work = import ./fs-work { inherit pkgs; }; - fs-backup = import ./fs-backup { inherit pkgs; }; + ekman-login = import ./ekman/login { inherit pkgs; }; + c0 = import ./ekman/c0 { inherit pkgs; }; + c0x = import ./ekman/c0x { inherit pkgs; }; + c1 = import ./ekman/c1 { inherit pkgs; }; + fs-work = import ./ekman/fs-work { inherit pkgs; }; + fs-backup = import ./ekman/fs-backup { inherit pkgs; }; in - { inherit ekman-manage; } // login // c0 // c0x // c1 // fs-work // fs-backup + { inherit ekman-manage; } // ekman-login // c0 // c0x // c1 // fs-work // fs-backup diff --git a/ekman/login/default.nix b/ekman/login/default.nix index 94baae8..29ce4cc 100644 --- a/ekman/login/default.nix +++ b/ekman/login/default.nix @@ -1,12 +1,11 @@ { pkgs ? import {} }: let - name = "rossby"; - address = "172.16.239.222"; - etcdCluster = import ../etcdCluster.nix; + name = "ekman"; + address = "10.255.241.100"; in { - rossby-login = { config, pkgs, ... }: with pkgs; { - deployment.tags = [ "rossby-login" "login" ]; + ekman = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "ekman" "login" ]; deployment.targetHost = address; system.autoUpgrade.enable = lib.mkForce false; @@ -21,13 +20,14 @@ in compute = true; k8sNode = true; mounts = { - rdma.enable = false; + rdma.enable = true; automount.enable = true; - users = false; + home = false; opt = false; - work = false; - data = false; - ceph = false; + work = true; + data = true; + backup = true; + ceph = true; }; }; @@ -40,17 +40,18 @@ in myvnc.enable = false; os = { - externalInterface = "enp129s0f0"; + externalInterface = "enp33s0f0np0"; nfs.enable = true; nfs.exports = '' - /exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) ''; }; hpc = { slurm.server = false; slurm.slurmrestd = false; - manager = false; + frontend = false; login = true; knem = false; }; @@ -58,7 +59,6 @@ in k8s = { master.enable = false; node.enable = true; - inherit etcdCluster; }; desktop.enable = false; @@ -66,7 +66,7 @@ in monitoring = { # server = { # enable = false; - # scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); + # scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); # defaultAlertReceiver = { # email_configs = [ # { to = "jonas.juselius@oceanbox.io"; } @@ -75,7 +75,7 @@ in # pageAlertReceiver = { # webhook_configs = [ # { - # url = "https://prometheus-msteams.k2.itpartner.no/rossby"; + # url = "https://prometheus-msteams.k2.itpartner.no/ekman"; # http_config = { # tls_config = { insecure_skip_verify = true; }; # }; @@ -93,7 +93,28 @@ in infiniband-exporter = { enable = true; nameMap = '' - # 0xe8ebd3030024981e "c0-1" + 0x248a07030029d2fc "frontend" + 0x248a07030029d104 "ekman" + 0x5aa2e1fffe1edfca "fs-work" + 0x1c34da0300787798 "fs-backup" + 0xe8ebd3030024981e "c0-1" + 0xe8ebd3030024a21a "c0-2" + 0xe8ebd30300249a3a "c0-3" + 0xe8ebd30300248b9e "c0-4" + 0xe8ebd30300248b86 "c0-5" + 0xe8ebd3030024998a "c0-6" + 0xe8ebd30300248b8e "c0-7" + 0xe8ebd3030024999e "c0-8" + 0xe8ebd30300248fca "c0-9" + 0xe8ebd3030024a216 "c0-10" + 0xe8ebd30300248b96 "c0-11" + 0xe8ebd30300248b9a "c0-12" + 0xe8ebd303002495d2 "c0-13" + 0xe8ebd303002495e2 "c0-14" + 0xe8ebd30300248f42 "c0-15" + 0xe8ebd303002495e6 "c0-16" + 0xe8ebd3030024a2a2 "c0-17" + 0xe8ebd3030024a2ae "c0-18" ''; }; slurm-exporter = { @@ -108,22 +129,15 @@ in # ''; # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6; - services.flannel.iface = "enp129s0f0"; + services.flannel.iface = "enp33s0f3np3"; networking = { useDHCP = false; hostName = name; -<<<<<<<< HEAD:rossby/login/default.nix - interfaces.enp129s0f0 = { - useDHCP = false; - ipv4.addresses = [ { - inherit address; -======== interfaces.enp33s0f3np3 = { useDHCP = false; ipv4.addresses = [ { address = address; ->>>>>>>> ekman:ekman/login/default.nix prefixLength = 24; } ]; # ipv4.routes = [ @@ -134,27 +148,27 @@ in # } # ]; }; - # interfaces."ibp65s0f0" = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = "10.255.243.100"; - # prefixLength = 24; - # } ]; - # }; - # interfaces."enp65s0f1np1" = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = "10.255.244.100"; - # prefixLength = 24; - # } ]; - # }; - # interfaces.enp33s0f0np0 = { - # useDHCP = false; - # ipv4.addresses = [ { - # address = "10.255.242.2"; - # prefixLength = 24; - # } ]; - # ipv4.routes = [ + interfaces."ibp65s0f0" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.100"; + prefixLength = 24; + } ]; + }; + interfaces."enp65s0f1np1" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.244.100"; + prefixLength = 24; + } ]; + }; + interfaces.enp33s0f0np0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.242.2"; + prefixLength = 24; + } ]; + ipv4.routes = [ # { # address = "10.1.8.0"; # prefixLength = 24; @@ -165,13 +179,15 @@ in # prefixLength = 24; # via = "10.255.242.1"; # } - # ]; - # }; - defaultGateway = "172.16.239.1"; + ]; + }; + defaultGateway = "10.255.242.1"; firewall = { allowedTCPPorts = [ 4443 ]; extraCommands = '' - # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + # needed for nodeport access on k1 and k2 + # iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.2 + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE ''; }; }; @@ -197,7 +213,7 @@ in device = "/vol/nfs-provisioner"; options = [ "bind" ]; }; - "/users" = { + "/frontend" = { device = "/home"; options = [ "bind" ]; }; @@ -212,7 +228,7 @@ in }; nix.extraOptions = '' - # secret-key-files = /etc/nix/rossby.key + secret-key-files = /etc/nix/ekman.key ''; # services.xserver = { @@ -232,8 +248,8 @@ in smtp_smarthost = "smtpgw.itpartner.no"; # smtp_auth_username = "utvikling"; # smtp_auth_password = "S0m3rp0m@de#21!"; - smtp_hello = "rossby.oceanbox.io"; - smtp_from = "noreplyrossby.oceanbox.io"; + smtp_hello = "ekman.oceanbox.io"; + smtp_from = "noreply@ekman.oceanbox.io"; }; # services.nginx = { @@ -266,7 +282,7 @@ in # sif = { # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; # executor = "shell"; - # tagList = [ "rossby" "sif" ]; + # tagList = [ "ekman" "sif" ]; # }; # }; # }; @@ -326,13 +342,14 @@ in "--login-server=https://headscale.svc.oceanbox.io" "--accept-dns" "--advertise-exit-node" - "--advertise-tags=tag:rossby" + "--advertise-routes=10.255.241.241.0/24" + "--advertise-tags=tag:ekman" ]; }; imports = [ ./hardware-configuration.nix - ../default.nix + ../cluster.nix ../mounts.nix ../myvnc.nix ]; diff --git a/ekman/login/hardware-configuration.nix b/ekman/login/hardware-configuration.nix index acc3e52..57814eb 100644 --- a/ekman/login/hardware-configuration.nix +++ b/ekman/login/hardware-configuration.nix @@ -8,20 +8,19 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-amd" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/e859992d-1627-42cc-a59c-178d596fc220"; + { device = "/dev/disk/by-uuid/c4a5ea80-2a87-440d-8431-a695d623da24"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/5281-9FFF"; + { device = "/dev/disk/by-uuid/D8A3-A04D"; fsType = "vfat"; - options = [ "fmask=0077" "dmask=0077" ]; }; swapDevices = [ ]; @@ -31,12 +30,10 @@ # still possible to use this option, but it's recommended to use it in conjunction # with explicit per-interface declarations with `networking.interfaces..useDHCP`. networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp129s0f0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp129s0f1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp129s0f2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp129s0f3.useDHCP = lib.mkDefault true; - # networking.interfaces.enp65s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true; - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/ekman/manage/default.nix b/ekman/manage/default.nix index b8065c5..af1da09 100644 --- a/ekman/manage/default.nix +++ b/ekman/manage/default.nix @@ -1,8 +1,3 @@ -<<<<<<< HEAD:configuration.nix -_: -{ - imports = [ ./rossby/manage ]; -======= { pkgs, ...}: let computeNodes = @@ -350,5 +345,5 @@ in { ../mounts.nix ../myvnc.nix ]; ->>>>>>> ekman:ekman/manage/default.nix } + diff --git a/ekman/mounts.nix b/ekman/mounts.nix index 73bfacc..fe38b06 100644 --- a/ekman/mounts.nix +++ b/ekman/mounts.nix @@ -3,19 +3,19 @@ with lib; let cfg = config.cluster.mounts; + subnet = + if cfg.rdma.enable then "243" + else if cfg.gbe100.enable then "244" + else "241"; + options = [ "soft" "defaults" "vers=4.2" ] ++ (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []); users = if cfg.users then { -<<<<<<<< HEAD:rossby/mounts.nix - "/users" = { - device = "172.16.239.222:/home"; -======== "/frontend" = { device = "10.255.241.100:/home"; ->>>>>>>> ekman:ekman/mounts.nix fsType = "nfs4"; options = [ "soft" @@ -28,7 +28,7 @@ let opt = let - server = "172.16.239.222"; + server = "10.255.241.100"; in if cfg.opt then { "/opt/bin" = { @@ -54,34 +54,49 @@ let device = "/ceph"; options = [ "bind" ]; }; + } else if cfg.backup then { + "/data" = { + device = "/backup"; + options = [ "bind" ]; + }; } else {}; + # device = "10.255.${subnet}.80:/backup"; + # # device = "10.255.${subnet}.80:/data"; + # fsType = "nfs4"; + # inherit options; + # }; work = if cfg.work then { "/work" = { - device = "172.16.240.210:/work"; + device = "10.255.${subnet}.90:/work"; fsType = "nfs4"; options = options ++ (if cfg.rdma.enable then [ "rdma" ] else []); }; } else {}; + backup = + if cfg.backup then { + "/backup" = { + device = "10.255.${subnet}.80:/backup"; + fsType = "nfs4"; + options = options ++ [ "ro" ] ++ (if cfg.rdma.enable then [ "rdma" ] else []); + }; + } else {}; + ceph = if cfg.ceph then { "/ceph" = { device = "oceanbox@.data=/"; fsType = "ceph"; options = [ - "mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789" + "mon_addr=10.255.241.30/10.255.241.31/10.255.241.32:6789" "_netdev" ]; }; } else {}; -<<<<<<<< HEAD:rossby/mounts.nix - fileSystems = users // opt // data // work // ceph; -======== fileSystems = users // opt // data // work // backup // ceph; ->>>>>>>> ekman:ekman/mounts.nix automount = mountpoint: if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then @@ -106,12 +121,12 @@ let "ceph/ceph.conf" = { text = '' [global] - mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789 + mon_host = 10.255.241.30:6789,10.255.241.31:6789,10.244.241.32:6789 log file = /tmp/ceph-$pid.log [client.oceanbox] - key = replaceme + key = AQDQNgRm6IE7JxAA1glJKsWPIBB/H/GxFYM0vQ== [client.rbd] - key = replaceme + key = AQCjth9mjR41ABAAvSs6hltidQT6Hu5OKwWu+Q== ''; mode = "0660"; group = "admin"; @@ -128,6 +143,7 @@ in opt = mkEnableOption "Enable /opt"; data = mkEnableOption "Enable /data"; work = mkEnableOption "Enable /work"; + backup = mkEnableOption "Enable /backup"; ceph = mkEnableOption "Enable /ceph"; };