diff --git a/cluster/compute/connauthfile b/cluster/c0/connauthfile similarity index 100% rename from cluster/compute/connauthfile rename to cluster/c0/connauthfile diff --git a/cluster/compute/default.nix b/cluster/c0/default.nix similarity index 86% rename from cluster/compute/default.nix rename to cluster/c0/default.nix index a93e18e..1ba8878 100644 --- a/cluster/compute/default.nix +++ b/cluster/c0/default.nix @@ -10,7 +10,7 @@ let nodes = import ./nodes.nix; compute = { - deployment.tags = [ "compute" ]; + deployment.tags = [ "compute" "c0" ]; fileSystems = { "/frontend" = { @@ -35,7 +35,6 @@ let mkCompute = host: let - ipoib = builtins.replaceStrings [".241."] [".243."] host.address; hw = ./hw + "/${host.name}.nix"; in { "${host.name}" = { @@ -56,9 +55,9 @@ let deployment.targetHost = host.address; - services.udev.extraRules = '' - KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" - ''; + # services.udev.extraRules = '' + # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; networking = { hostName = host.name; @@ -76,10 +75,11 @@ let } ]; }; - interfaces."ibp1s0.7666" = { + # interfaces."ibp1s0.7666" = { + interfaces."ibp1s0" = { useDHCP = false; ipv4.addresses = [ { - address = ipoib; + address = host.ipoib; prefixLength = 24; } ]; }; diff --git a/cluster/compute/hw/c0-1.nix b/cluster/c0/hw/c0-1.nix similarity index 100% rename from cluster/compute/hw/c0-1.nix rename to cluster/c0/hw/c0-1.nix diff --git a/cluster/compute/hw/c0-10.nix b/cluster/c0/hw/c0-10.nix similarity index 100% rename from cluster/compute/hw/c0-10.nix rename to cluster/c0/hw/c0-10.nix diff --git a/cluster/compute/hw/c0-11.nix b/cluster/c0/hw/c0-11.nix similarity index 100% rename from cluster/compute/hw/c0-11.nix rename to cluster/c0/hw/c0-11.nix diff --git a/cluster/compute/hw/c0-12.nix b/cluster/c0/hw/c0-12.nix similarity index 100% rename from cluster/compute/hw/c0-12.nix rename to cluster/c0/hw/c0-12.nix diff --git a/cluster/compute/hw/c0-13.nix b/cluster/c0/hw/c0-13.nix similarity index 100% rename from cluster/compute/hw/c0-13.nix rename to cluster/c0/hw/c0-13.nix diff --git a/cluster/compute/hw/c0-14.nix b/cluster/c0/hw/c0-14.nix similarity index 100% rename from cluster/compute/hw/c0-14.nix rename to cluster/c0/hw/c0-14.nix diff --git a/cluster/compute/hw/c0-15.nix b/cluster/c0/hw/c0-15.nix similarity index 100% rename from cluster/compute/hw/c0-15.nix rename to cluster/c0/hw/c0-15.nix diff --git a/cluster/compute/hw/c0-16.nix b/cluster/c0/hw/c0-16.nix similarity index 100% rename from cluster/compute/hw/c0-16.nix rename to cluster/c0/hw/c0-16.nix diff --git a/cluster/compute/hw/c0-2.nix b/cluster/c0/hw/c0-2.nix similarity index 100% rename from cluster/compute/hw/c0-2.nix rename to cluster/c0/hw/c0-2.nix diff --git a/cluster/compute/hw/c0-3.nix b/cluster/c0/hw/c0-3.nix similarity index 100% rename from cluster/compute/hw/c0-3.nix rename to cluster/c0/hw/c0-3.nix diff --git a/cluster/compute/hw/c0-4.nix b/cluster/c0/hw/c0-4.nix similarity index 100% rename from cluster/compute/hw/c0-4.nix rename to cluster/c0/hw/c0-4.nix diff --git a/cluster/compute/hw/c0-5.nix b/cluster/c0/hw/c0-5.nix similarity index 100% rename from cluster/compute/hw/c0-5.nix rename to cluster/c0/hw/c0-5.nix diff --git a/cluster/compute/hw/c0-6.nix b/cluster/c0/hw/c0-6.nix similarity index 100% rename from cluster/compute/hw/c0-6.nix rename to cluster/c0/hw/c0-6.nix diff --git a/cluster/compute/hw/c0-7.nix b/cluster/c0/hw/c0-7.nix similarity index 100% rename from cluster/compute/hw/c0-7.nix rename to cluster/c0/hw/c0-7.nix diff --git a/cluster/compute/hw/c0-8.nix b/cluster/c0/hw/c0-8.nix similarity index 100% rename from cluster/compute/hw/c0-8.nix rename to cluster/c0/hw/c0-8.nix diff --git a/cluster/compute/hw/c0-9.nix b/cluster/c0/hw/c0-9.nix similarity index 100% rename from cluster/compute/hw/c0-9.nix rename to cluster/c0/hw/c0-9.nix diff --git a/cluster/compute/kernel.nix b/cluster/c0/kernel.nix similarity index 100% rename from cluster/compute/kernel.nix rename to cluster/c0/kernel.nix diff --git a/cluster/c0/nodes.nix b/cluster/c0/nodes.nix new file mode 100644 index 0000000..752014b --- /dev/null +++ b/cluster/c0/nodes.nix @@ -0,0 +1,11 @@ +with builtins; +let + nodes = genList (n: n + 1) 16; in +map (n: ( + rec { + idx = 100 + n; + name = "c0-${toString n}"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ./. + "/pubkeys/c0-${toString n}.pub"; + })) nodes diff --git a/cluster/compute/pubkeys/beegfs0.pub b/cluster/c0/pubkeys/beegfs0.pub similarity index 100% rename from cluster/compute/pubkeys/beegfs0.pub rename to cluster/c0/pubkeys/beegfs0.pub diff --git a/cluster/compute/pubkeys/c0-1.pub b/cluster/c0/pubkeys/c0-1.pub similarity index 100% rename from cluster/compute/pubkeys/c0-1.pub rename to cluster/c0/pubkeys/c0-1.pub diff --git a/cluster/compute/pubkeys/c0-10.pub b/cluster/c0/pubkeys/c0-10.pub similarity index 100% rename from cluster/compute/pubkeys/c0-10.pub rename to cluster/c0/pubkeys/c0-10.pub diff --git a/cluster/compute/pubkeys/c0-11.pub b/cluster/c0/pubkeys/c0-11.pub similarity index 100% rename from cluster/compute/pubkeys/c0-11.pub rename to cluster/c0/pubkeys/c0-11.pub diff --git a/cluster/compute/pubkeys/c0-12.pub b/cluster/c0/pubkeys/c0-12.pub similarity index 100% rename from cluster/compute/pubkeys/c0-12.pub rename to cluster/c0/pubkeys/c0-12.pub diff --git a/cluster/compute/pubkeys/c0-13.pub b/cluster/c0/pubkeys/c0-13.pub similarity index 100% rename from cluster/compute/pubkeys/c0-13.pub rename to cluster/c0/pubkeys/c0-13.pub diff --git a/cluster/compute/pubkeys/c0-14.pub b/cluster/c0/pubkeys/c0-14.pub similarity index 100% rename from cluster/compute/pubkeys/c0-14.pub rename to cluster/c0/pubkeys/c0-14.pub diff --git a/cluster/compute/pubkeys/c0-15.pub b/cluster/c0/pubkeys/c0-15.pub similarity index 100% rename from cluster/compute/pubkeys/c0-15.pub rename to cluster/c0/pubkeys/c0-15.pub diff --git a/cluster/compute/pubkeys/c0-16.pub b/cluster/c0/pubkeys/c0-16.pub similarity index 100% rename from cluster/compute/pubkeys/c0-16.pub rename to cluster/c0/pubkeys/c0-16.pub diff --git a/cluster/compute/pubkeys/c0-2.pub b/cluster/c0/pubkeys/c0-2.pub similarity index 100% rename from cluster/compute/pubkeys/c0-2.pub rename to cluster/c0/pubkeys/c0-2.pub diff --git a/cluster/compute/pubkeys/c0-3.pub b/cluster/c0/pubkeys/c0-3.pub similarity index 100% rename from cluster/compute/pubkeys/c0-3.pub rename to cluster/c0/pubkeys/c0-3.pub diff --git a/cluster/compute/pubkeys/c0-4.pub b/cluster/c0/pubkeys/c0-4.pub similarity index 100% rename from cluster/compute/pubkeys/c0-4.pub rename to cluster/c0/pubkeys/c0-4.pub diff --git a/cluster/compute/pubkeys/c0-5.pub b/cluster/c0/pubkeys/c0-5.pub similarity index 100% rename from cluster/compute/pubkeys/c0-5.pub rename to cluster/c0/pubkeys/c0-5.pub diff --git a/cluster/compute/pubkeys/c0-6.pub b/cluster/c0/pubkeys/c0-6.pub similarity index 100% rename from cluster/compute/pubkeys/c0-6.pub rename to cluster/c0/pubkeys/c0-6.pub diff --git a/cluster/compute/pubkeys/c0-7.pub b/cluster/c0/pubkeys/c0-7.pub similarity index 100% rename from cluster/compute/pubkeys/c0-7.pub rename to cluster/c0/pubkeys/c0-7.pub diff --git a/cluster/compute/pubkeys/c0-8.pub b/cluster/c0/pubkeys/c0-8.pub similarity index 100% rename from cluster/compute/pubkeys/c0-8.pub rename to cluster/c0/pubkeys/c0-8.pub diff --git a/cluster/compute/pubkeys/c0-9.pub b/cluster/c0/pubkeys/c0-9.pub similarity index 100% rename from cluster/compute/pubkeys/c0-9.pub rename to cluster/c0/pubkeys/c0-9.pub diff --git a/cluster/c1/connauthfile b/cluster/c1/connauthfile new file mode 100644 index 0000000..37cd965 --- /dev/null +++ b/cluster/c1/connauthfile @@ -0,0 +1 @@ +ä·q‹u \ No newline at end of file diff --git a/cluster/c1/default.nix b/cluster/c1/default.nix new file mode 100644 index 0000000..a3e8aa3 --- /dev/null +++ b/cluster/c1/default.nix @@ -0,0 +1,91 @@ +{ pkgs ? import {} }: +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; + # }) {}; + # pkgs = import {}; + + nodes = import ./nodes.nix; + + compute = { + deployment.tags = [ "compute" "c1" ]; + + fileSystems = { + "/frontend" = { + device = "10.255.241.100:/home"; + fsType = "nfs4"; + options = [ + "soft" + "defaults" + "noauto" + "x-systemd.automount" + ]; + }; + }; + + systemd.automounts = [ + { + where = "/frontend"; + wantedBy = [ "default.target" ]; + } + ]; + }; + + mkCompute = host: + let + hw = ./hw + "/${host.name}.nix"; + in { + "${host.name}" = { + cluster = { + compute = true; + k8sNode = true; + }; + + features = { + host = { + name = host.name; + address = host.address; + }; + os.externalInterface = "eno33"; + hpc.compute = true; + # k8s = { inherit etcdCluster; }; + }; + + deployment.targetHost = host.address; + + # services.udev.extraRules = '' + # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + networking = { + hostName = host.name; + useDHCP = false; + interfaces.eno33 = { + useDHCP = false; + ipv4.addresses = [ { + address = host.address; + prefixLength = 24; + } ]; + ipv4.routes = [ { + address = "10.255.242.2"; + prefixLength = 32; + via = "10.255.241.100"; + } ]; + + }; + interfaces.ibp65s0 = { + useDHCP = false; + ipv4.addresses = [ { + address = host.ipoib; + prefixLength = 24; + } ]; + }; + }; + imports = [ ../cluster.nix hw ]; + } + // compute; +}; +in builtins.foldl' (a: n: a // mkCompute n) {} nodes + diff --git a/cluster/c1/hw/c1-1.nix b/cluster/c1/hw/c1-1.nix new file mode 100644 index 0000000..0179b2c --- /dev/null +++ b/cluster/c1/hw/c1-1.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/3af91585-8079-420d-acdf-f60b94d3cfff"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/3590-199A"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-2.nix b/cluster/c1/hw/c1-2.nix new file mode 100644 index 0000000..1f638eb --- /dev/null +++ b/cluster/c1/hw/c1-2.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/d89e1496-fda1-4de0-b2cc-474967b04402"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/A51A-1F4D"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-3.nix b/cluster/c1/hw/c1-3.nix new file mode 100644 index 0000000..c98b526 --- /dev/null +++ b/cluster/c1/hw/c1-3.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/84cc9cea-08eb-4b54-8ca3-2aa5c1300a92"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/882D-A342"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-4.nix b/cluster/c1/hw/c1-4.nix new file mode 100644 index 0000000..c34903f --- /dev/null +++ b/cluster/c1/hw/c1-4.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f0826ad5-8a4e-427d-98d3-5afa44440993"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/21A6-D34C"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-5.nix b/cluster/c1/hw/c1-5.nix new file mode 100644 index 0000000..43ad87f --- /dev/null +++ b/cluster/c1/hw/c1-5.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/37944fce-07ca-492f-906a-620a37e7e1b3"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/2F51-EC20"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-6.nix b/cluster/c1/hw/c1-6.nix new file mode 100644 index 0000000..ada6f39 --- /dev/null +++ b/cluster/c1/hw/c1-6.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/d23386a8-0ca8-4871-b662-decf2b24f4d7"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/AF8A-DEFE"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-7.nix b/cluster/c1/hw/c1-7.nix new file mode 100644 index 0000000..54107e3 --- /dev/null +++ b/cluster/c1/hw/c1-7.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/12477966-c6c5-47c6-afdc-35fa7e57e837"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/DB96-7453"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/hw/c1-8.nix b/cluster/c1/hw/c1-8.nix new file mode 100644 index 0000000..d4dedcc --- /dev/null +++ b/cluster/c1/hw/c1-8.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/2f468316-5832-4684-866d-2e92b08fb68b"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/76FE-F657"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/cluster/c1/kernel.nix b/cluster/c1/kernel.nix new file mode 100644 index 0000000..750da0d --- /dev/null +++ b/cluster/c1/kernel.nix @@ -0,0 +1,46 @@ +{pkgs, lib, stdenv, fetchurl, config, kernel ? pkgs.linux, ...}: +let + i40e = + stdenv.mkDerivation rec { + name = "i40e-${version}-${kernel.version}"; + version = "2.13.10"; + + src = pkgs.fetchFromGitHub { + owner = "dmarion"; + repo = "i40e"; + rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd"; + sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr"; + }; + + hardeningDisable = [ "pic" ]; + + configurePhase = '' + cd src + kernel_version=${kernel.modDirVersion} + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk + export makeFlags="BUILD_KERNEL=$kernel_version" + ''; + + installPhase = '' + install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko" + ''; + + dontStrip = true; + + enableParallelBuilding = true; + + meta = { + description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)"; + homepage = https://github.com/dmarion/i40e; + license = lib.licenses.gpl2; + }; + }; +in +{ + i40e2 = i40e; + overlay = self: super: { + # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; + }; +} + diff --git a/cluster/c1/nodes.nix b/cluster/c1/nodes.nix new file mode 100644 index 0000000..b7effe9 --- /dev/null +++ b/cluster/c1/nodes.nix @@ -0,0 +1,10 @@ +with builtins; +let nodes = genList (n: n + 1) 8; in +map (n: ( + rec { + idx = 120 + n; + name = "c1-${toString n}"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ./. + "/pubkeys/c1-${toString n}.pub"; + })) nodes diff --git a/cluster/c1/pubkeys/c1-1.pub b/cluster/c1/pubkeys/c1-1.pub new file mode 100644 index 0000000..1351c52 --- /dev/null +++ b/cluster/c1/pubkeys/c1-1.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwwu4RJJhKo8s2Mtmpdvxs02d4IwrmS9cTpfAOQ2YkI root@nixos diff --git a/cluster/c1/pubkeys/c1-2.pub b/cluster/c1/pubkeys/c1-2.pub new file mode 100644 index 0000000..c09679b --- /dev/null +++ b/cluster/c1/pubkeys/c1-2.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBEV8wEkeGUOs6umhdeOKYnVlYlta2rOCZSoezvu+bZ4 root@nixos diff --git a/cluster/c1/pubkeys/c1-3.pub b/cluster/c1/pubkeys/c1-3.pub new file mode 100644 index 0000000..0c58f4b --- /dev/null +++ b/cluster/c1/pubkeys/c1-3.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQFQqo2vcj2eYaH7nwdEzgCPme+7g3Db+s16KbQHzLI root@nixos diff --git a/cluster/c1/pubkeys/c1-4.pub b/cluster/c1/pubkeys/c1-4.pub new file mode 100644 index 0000000..df6aaee --- /dev/null +++ b/cluster/c1/pubkeys/c1-4.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDIsqJqWWftoiyiOOHnIuHYqbsaOg4AbKNm80wpjH2E8 root@nixos diff --git a/cluster/c1/pubkeys/c1-5.pub b/cluster/c1/pubkeys/c1-5.pub new file mode 100644 index 0000000..c746bf4 --- /dev/null +++ b/cluster/c1/pubkeys/c1-5.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDme/xztp22wQOybtN2TVXMcn2QcVaXtRMp4AnPnzr2T root@nixos diff --git a/cluster/c1/pubkeys/c1-6.pub b/cluster/c1/pubkeys/c1-6.pub new file mode 100644 index 0000000..b64a074 --- /dev/null +++ b/cluster/c1/pubkeys/c1-6.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImItUKMSygMY/1ZNsyGkyfywyngEDgZ7TxM63UwG1VH root@nixos diff --git a/cluster/c1/pubkeys/c1-7.pub b/cluster/c1/pubkeys/c1-7.pub new file mode 100644 index 0000000..1111820 --- /dev/null +++ b/cluster/c1/pubkeys/c1-7.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID1WxWnNg+L2+lQ3W2mD0/6kqwuCUOEJImvWnQYLzdUB root@nixos diff --git a/cluster/c1/pubkeys/c1-8.pub b/cluster/c1/pubkeys/c1-8.pub new file mode 100644 index 0000000..5c92679 --- /dev/null +++ b/cluster/c1/pubkeys/c1-8.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBq+v2bBhQieqfXmtmGYb/9mOv9oc88zerRkkpGpoc0x root@nixos diff --git a/cluster/c1/pubkeys/frontend.pub b/cluster/c1/pubkeys/frontend.pub new file mode 100644 index 0000000..b4c0d9c --- /dev/null +++ b/cluster/c1/pubkeys/frontend.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkSSuHkieXwgFMKRy4MjwjNrJEItWbQHeAAH+Zn1YZp root@localhost diff --git a/cluster/cluster.nix b/cluster/cluster.nix index 46be353..a960075 100644 --- a/cluster/cluster.nix +++ b/cluster/cluster.nix @@ -3,7 +3,9 @@ with lib; let cfg = config.features.host; - computeNodes = import ./compute/nodes.nix; + computeNodes = + import ./c0/nodes.nix ++ + import ./c1/nodes.nix; mkSANs = host: [ host.name @@ -85,6 +87,7 @@ let "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5k0dXn60dZ3iORy99LVvgTldu9nYU1TJVL1wCJEqp kaih kubernetes" ]; docker.enable = false; }; @@ -114,11 +117,12 @@ let # gid = 0; # }; - nix = { - maxJobs = 32; - trustedUsers = [ "@wheel" ]; - binaryCachePublicKeys = [ - "ekman.local:2NsTThGkZVJtOs3NVQYjEZ4NLscXlbjqA8Fi7HnAreA=" + services.certmgr.validMin = "120h"; + + nix.settings = { + max-jobs = 32; + trusted-users = [ "@wheel" ]; + substituters = [ ]; }; }; @@ -127,21 +131,25 @@ let features.hpc.slurm = { enable = true; client = true; - mungeKey = ./compute/munge.key; + mungeKey = ./munge.key; mungeUid = mkDefault 996; # hack - pkey = "0x7666"; - controlMachine = "ekman"; + # pkey = "0x7666"; + controlMachine = "frontend"; mailDomain = "oceanbox.io"; nodeName = [ "c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" - "nfs0 Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" - "nfs1 Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" + "fs0 Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "fs1 Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" ]; partitionName = [ "batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP" "frontend Nodes=ekman MaxTime=1:00:00 State=UP" - "short Nodes=nfs[0-1] MaxTime=1:00:00 State=UP" + "short Nodes=c1-[1-8] MaxTime=1:00:00 State=UP" + "medium Nodes=c1-[1-8] MaxTime=4:00:00 State=UP" + "longish Nodes=c1-[1-8] MaxTime=8:00:00 State=UP" + "long Nodes=c1-[1-8] MaxTime=72:00:00 State=UP" ]; }; }; @@ -183,6 +191,11 @@ let fsType = "nfs"; options = [ "soft" "rdma" "defaults" "vers=4.2" ]; }; + "/backup" = { + device = "10.255.243.81:/backup"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; + }; "/work" = { device = "10.255.243.90:/work"; fsType = "nfs"; @@ -200,14 +213,16 @@ let initca = ./ca; cidr = "10.100.0.0/16"; master = { - name = "ekman"; - address = "10.255.241.100"; - extraSANs = [ "ekman.local" "ekman.compute.local" "ekman.oceanbox.io" ]; + name = "frontend"; + address = "10.255.241.99"; + extraSANs = [ + "frontend.local" "frontend.compute.local" "frontend.oceanbox.io" + ]; }; ingressNodes = [ "ekman.oceanbox.io" ]; - fileserver = "ibnfs0"; + fileserver = "ibfs0"; charts = { acme_email = "acme@oceanbox.io"; # grafana_smtp_user = "utvikling"; @@ -238,41 +253,54 @@ let }; programs.ssh.knownHosts = { + frontend = { + hostNames = [ + "frontend" "frontend.compute.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.90" + ]; + publicKeyFile = ../frontend.pub; + }; ekman = { hostNames = [ "ekman" "ekman.compute.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100" ]; - publicKeyFile = ../ekman.pub; + publicKeyFile = ./ekman/ekman.pub; }; - nfs0 = { + fs0 = { hostNames = [ - "nfs0" "nfs0.compute.local" "10.255.241.80" "10.255.243.80" + "fs0" "fs0.compute.local" "10.255.241.80" "10.255.243.80" ]; - publicKeyFile = ./nfs0/nfs0.pub; + publicKeyFile = ./fs0/fs0.pub; }; - nfs1 = { + fs1 = { hostNames = [ - "nfs1" "nfs1.compute.local" "10.255.241.90" "10.255.243.90" + "fs1" "fs1.compute.local" "10.255.241.90" "10.255.243.90" ]; - publicKeyFile = ./nfs1/nfs1.pub; + publicKeyFile = ./fs1/fs1.pub; + }; + fs2 = { + hostNames = [ + "fs2" "fs2.compute.local" "10.255.241.81" "10.255.243.81" + ]; + publicKeyFile = ./fs2/fs2.pub; }; } // builtins.foldl' (a: x: - let - n = toString x.idx; - addr = toString (x.idx + 100); + let n = toString x.idx; in a // { - "c0-${n}" = { + "${x.name}" = { hostNames = [ - "c0-${n}" - "c0-${n}.compute.local" - "10.255.241.${addr}" - "10.255.243.${addr}" + "${x.name}" + "${x.name}.compute.local" + "10.255.241.${n}" + "10.255.243.${n}" ]; - publicKeyFile = ./compute/pubkeys/c0-${n}.pub; + publicKeyFile = x.pubkey; }; }) {} computeNodes; - environment.systemPackages = [ openssh-shosts ]; + environment.systemPackages = [ + openssh-shosts + pkgs.inotify-tools + ]; security.wrappers = { ssh-keysign = { diff --git a/cluster/compute/nodes.nix b/cluster/compute/nodes.nix deleted file mode 100644 index 01b0128..0000000 --- a/cluster/compute/nodes.nix +++ /dev/null @@ -1,3 +0,0 @@ -with builtins; -let nodes = genList (n: n + 1) 16; in -map (n: ({ idx = n; name = "c0-${toString n}"; address = "10.255.241.${toString (n + 100)}"; })) nodes diff --git a/cluster/ekman/default.nix b/cluster/ekman/default.nix new file mode 100644 index 0000000..e7eeb86 --- /dev/null +++ b/cluster/ekman/default.nix @@ -0,0 +1,302 @@ +{ pkgs ? import {} }: +let + name = "ekman"; + address = "10.255.241.100"; +in +{ + ekman = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "ekman" "login" ]; + deployment.targetHost = address; + system.autoUpgrade.enable = lib.mkForce false; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + cluster = { + compute = true; + k8sNode = true; + }; + + features = { + host = { + name = "ekman"; + address = "10.255.241.100"; + }; + + myvnc.enable = true; + + os = { + externalInterface = "enp33s0f0np0"; + nfs.enable = true; + nfs.exports = '' + /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + hpc = { + slurm.server = false; + slurm.slurmrestd = true; + frontend = false; + login = true; + }; + + k8s = { + master.enable = false; + node.enable = true; + }; + + # monitoring = { + # server = { + # enable = false; + # scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); + # defaultAlertReceiver = { + # email_configs = [ + # { to = "jonas.juselius@oceanbox.io"; } + # ]; + # }; + # pageAlertReceiver = { + # webhook_configs = [ + # { + # url = "https://prometheus-msteams.k2.itpartner.no/ekman"; + # http_config = { + # tls_config = { insecure_skip_verify = true; }; + # }; + # } + # ]; + # }; + # }; + # webUI.enable = false; + # webUI.acmeEmail = "innovasjon@itpartner.no"; + # webUI.allow = [ + # "10.1.2.0/24" + # "172.19.254.0/24" + # "172.19.255.0/24" + # ]; + # infiniband-exporter = { + # enable = true; + # nameMap = '' + # 0xe8ebd3030024a2c6 "frontend" + # 0xe8ebd3030024981e "c0-1" + # 0xe8ebd3030024a21a "c0-2" + # 0xe8ebd30300249a3a "c0-3" + # 0xe8ebd30300248b9e "c0-4" + # 0xe8ebd30300248b86 "c0-5" + # 0xe8ebd3030024998a "c0-6" + # 0xe8ebd30300248b8e "c0-7" + # 0xe8ebd3030024999e "c0-8" + # 0xe8ebd30300248fca "c0-9" + # 0xe8ebd3030024a216 "c0-10" + # 0xe8ebd30300248b96 "c0-11" + # 0xe8ebd30300248b9a "c0-12" + # 0xe8ebd303002495d2 "c0-13" + # 0xe8ebd303002495e2 "c0-14" + # 0xe8ebd30300248f42 "c0-15" + # 0xe8ebd303002495e6 "c0-16" + # 0xe8ebd3030024a2a2 "nfs0" + # 0xe8ebd3030024a2ae "nfs1" + # 0x0c42a10300ddc4bc "stokes" + # 0x1c34da0300787798 "mds0-0" + # 0x0c42a10300dbe7f4 "c1-1" + # 0x0c42a10300dbe7d8 "c1-2" + # 0x0c42a10300dbe800 "c1-3" + # 0x0c42a10300dbec80 "c1-4" + # 0x0c42a10300dbea50 "c1-5" + # 0x0c42a10300dbeb2c "c1-6" + # 0x0c42a10300dbe7fc "c1-7" + # 0x0c42a10300dbe5a0 "c1-8" + # ''; + # }; + # slurm-exporter = { + # enable = true; + # port = 6080; + # }; + # }; + }; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + networking = { + useDHCP = false; + hostName = "ekman"; + interfaces.enp33s0f0np0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.242.2"; + prefixLength = 24; + } ]; + ipv4.routes = [ + { + address = "10.1.8.0"; + prefixLength = 24; + via = "10.255.242.1"; + } + { + address = "10.1.30.0"; + prefixLength = 24; + via = "10.255.242.1"; + } + ]; + }; + interfaces.enp33s0f3np3 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.241.100"; + prefixLength = 24; + } ]; + }; + interfaces."ibp65s0" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.100"; + prefixLength = 24; + } ]; + }; + defaultGateway = "10.255.242.1"; + firewall = { + allowedTCPPorts = [ 4443 ]; + extraCommands = '' + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT + # needed for nodeport access on k1 and k2 + iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.241.0/24 -j SNAT --to-source 10.255.242.2 + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; + }; + + fileSystems = { + "/exports/home" = { + device = "/home"; + options = [ "bind" ]; + }; + "/frontend" = { + device = "/home"; + options = [ "bind" ]; + }; + "/vol/local-storage/vol1" = { + device = "/vol/vol1"; + options = [ "bind" ]; + }; + "/vol/local-storage/vol2" = { + device = "/vol/vol2"; + options = [ "bind" ]; + }; + }; + + nix.extraOptions = '' + secret-key-files = /etc/nix/ekman.key + ''; + + services.xserver = { + enable = true; + enableCtrlAltBackspace = true; + layout = "us"; + xkbVariant = "altgr-intl"; + xkbOptions = "eurosign:e"; + displayManager = { + gdm.enable = true; + job.logToFile = true; + }; + # desktopManager.xfce.enable = true; + }; + + services.prometheus.alertmanager.configuration.global = { + smtp_smarthost = "smtpgw.itpartner.no"; + # smtp_auth_username = "utvikling"; + # smtp_auth_password = "S0m3rp0m@de#21!"; + smtp_hello = "ekman.oceanbox.io"; + smtp_from = "noreply@ekman.oceanbox.io"; + }; + + # services.nginx = { + # virtualHosts = { + # "ds.matnoc.regnekraft.io" = { + # forceSSL = true; + # enableACME = true; + # serverAliases = []; + # locations."/" = { + # proxyPass = "http://localhost:9088"; + # proxyWebsockets = false; + # extraConfig = '' + # allow 10.1.2.0/24; + # allow 172.19.254.0/24; + # allow 172.19.255.0/24; + # deny all; + # ''; + # }; + # }; + # }; + # }; + + # services.gitlab-runner = { + # enable = true; + # extraPackages = with pkgs; [ + # singularity + # ]; + # concurrent = 4; + # services = { + # sif = { + # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; + # executor = "shell"; + # tagList = [ "ekman" "sif" ]; + # }; + # }; + # }; + + # security.sudo.extraConfig = '' + # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity + # ''; + + security.pam = { + services.sshd.googleAuthenticator.enable = true; + loginLimits = [ + { + domain = "@users"; + item = "rss"; + type = "hard"; + value = 16000000; + } + { + domain = "@users"; + item = "cpu"; + type = "hard"; + value = 180; + } + ]; + }; + + system.activationScripts = { + home-permissions.text = '' + chmod 755 /home/olean + chmod 755 /home/frankgaa + chmod 755 /home/jonas + chmod 755 /home/stig + chmod 755 /home/bast + chmod 755 /home/simenlk + ''; + }; + + + # ssh-rsa is deprecated, but putty/winscp users use it + services.openssh.extraConfig = '' + # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 + PubkeyAuthOptions verify-required + ''; + + environment.systemPackages = []; + + imports = [ + ../cluster.nix + ../myvnc.nix + ./hardware-configuration.nix + ]; + }; +} + diff --git a/ekman.pub b/cluster/ekman/ekman.pub similarity index 100% rename from ekman.pub rename to cluster/ekman/ekman.pub diff --git a/cluster/ekman/hardware-configuration.nix b/cluster/ekman/hardware-configuration.nix new file mode 100644 index 0000000..57814eb --- /dev/null +++ b/cluster/ekman/hardware-configuration.nix @@ -0,0 +1,39 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/c4a5ea80-2a87-440d-8431-a695d623da24"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/D8A3-A04D"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/etcdCluster.nix b/cluster/etcdCluster.nix index 44f6af6..8565caf 100644 --- a/cluster/etcdCluster.nix +++ b/cluster/etcdCluster.nix @@ -3,9 +3,10 @@ existing = true; nodes = { + frontend = "10.255.241.99"; ekman = "10.255.241.100"; - nfs0 = "10.255.241.80"; - nfs1 = "10.255.241.90"; + fs0 = "10.255.241.80"; + fs1 = "10.255.241.90"; }; } diff --git a/cluster/nfs0/default.nix b/cluster/fs0/default.nix similarity index 81% rename from cluster/nfs0/default.nix rename to cluster/fs0/default.nix index 0fba359..36b2c5e 100644 --- a/cluster/nfs0/default.nix +++ b/cluster/fs0/default.nix @@ -10,7 +10,7 @@ let name = "nfs0"; address = "10.255.241.80"; in { - nfs0 = { config, pkgs, ... }: with pkgs; { + fs0 = { config, pkgs, ... }: with pkgs; { deployment.tags = [ "fs" "fs-data" ]; deployment.targetHost = address; system.autoUpgrade.enable = lib.mkForce false; @@ -22,9 +22,9 @@ in { hybrid-sleep.enable = false; }; - services.udev.extraRules = '' - KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" - ''; + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; environment.systemPackages = with pkgs; [ rdma-core @@ -33,7 +33,7 @@ in { cluster = { k8sNode = true; - slurm = true; + slurm = false; }; features.hpc.slurm.mungeUid = 994; @@ -114,11 +114,11 @@ in { } ]; }; + # interfaces."ibp65s0.7666" = { + # useDHCP = false; + # }; interfaces."ibp65s0" = { useDHCP = false; - }; - interfaces."ibp65s0.7666" = { - useDHCP = false; ipv4.addresses = [ { address = "10.255.243.80"; prefixLength = 24; @@ -157,6 +157,11 @@ in { fsType = "nfs"; options = [ "soft" "rdma" "defaults" "vers=4.2" ]; }; + "/backup" = { + device = "10.255.243.81:/backup"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; + }; }; environment.etc = { @@ -179,6 +184,28 @@ in { dataDir = [ "/data/s3" ]; }; + services.postgresql = { + enable = true; + package = pkgs.postgresql_16; + dataDir = "/data/postgresql/data"; + enableTCPIP = true; + identMap = '' + nixos root postgres + nixos admin postgres + ''; + authentication = pkgs.lib.mkOverride 11 '' + local all all trust + host all all ::1/128 trust + host all all ::1/128 md5 + host all postgres 127.0.0.1/32 md5 + host all postgres ::1/128 md5 + host all postgres 10.255.241.0/24 md5 + host all postgres 10.255.243.0/24 md5 + host archivist archivist 10.255.241.0/24 md5 + host archivist archivist 10.255.243.0/24 md5 + ''; + }; + programs.singularity.enable = true; imports = [ diff --git a/cluster/nfs0/nfs0.pub b/cluster/fs0/fs0.pub similarity index 100% rename from cluster/nfs0/nfs0.pub rename to cluster/fs0/fs0.pub diff --git a/cluster/nfs0/hardware-configuration.nix b/cluster/fs0/hardware-configuration.nix similarity index 100% rename from cluster/nfs0/hardware-configuration.nix rename to cluster/fs0/hardware-configuration.nix diff --git a/cluster/nfs1/default.nix b/cluster/fs1/default.nix similarity index 94% rename from cluster/nfs1/default.nix rename to cluster/fs1/default.nix index fa5ae73..b64f246 100644 --- a/cluster/nfs1/default.nix +++ b/cluster/fs1/default.nix @@ -10,7 +10,7 @@ let name = "nfs1"; address = "10.255.241.90"; in { - nfs1 = { config, pkgs, ... }: with pkgs; { + fs1 = { config, pkgs, ... }: with pkgs; { deployment.tags = [ "fs" "fs-work" ]; deployment.targetHost = address; system.autoUpgrade.enable = lib.mkForce false; @@ -22,9 +22,9 @@ in { hybrid-sleep.enable = false; }; - services.udev.extraRules = '' - KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" - ''; + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; environment.systemPackages = with pkgs; [ rdma-core @@ -33,7 +33,7 @@ in { cluster = { k8sNode = true; - slurm = true; + slurm = false; }; features.hpc.slurm.mungeUid = 994; @@ -114,11 +114,11 @@ in { } ]; }; + # interfaces."ibp65s0.7666" = { + # useDHCP = false; + # }; interfaces."ibp65s0" = { useDHCP = false; - }; - interfaces."ibp65s0.7666" = { - useDHCP = false; ipv4.addresses = [ { address = "10.255.243.90"; prefixLength = 24; diff --git a/cluster/nfs1/nfs1.pub b/cluster/fs1/fs1.pub similarity index 100% rename from cluster/nfs1/nfs1.pub rename to cluster/fs1/fs1.pub diff --git a/cluster/nfs1/hardware-configuration.nix b/cluster/fs1/hardware-configuration.nix similarity index 100% rename from cluster/nfs1/hardware-configuration.nix rename to cluster/fs1/hardware-configuration.nix diff --git a/cluster/fs2/default.nix b/cluster/fs2/default.nix new file mode 100644 index 0000000..722ff1c --- /dev/null +++ b/cluster/fs2/default.nix @@ -0,0 +1,223 @@ +{ pkgs ? import {} }: +let + name = "fs2"; + address = "10.255.241.81"; +in { + fs2 = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "fs" "fs-backup" ]; + deployment.targetHost = address; + system.autoUpgrade.enable = lib.mkForce false; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + xfsprogs + ]; + + boot.swraid = { + enable = true; + mdadmConf = '' + DEVICE partitions + ARRAY /dev/md/0 metadata=1.2 UUID=b743fdd4:5b339cc7:7c43f50f:3b81243e name=fs2:0 + ''; + }; + + systemd.services.restart-md0 = { + description = "restart /dev/md0"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-devices-virtual-block-md0.device" "-.mount" ]; + before = [ "backup.mount" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' + restart=0 + ${util-linux}/bin/lsblk -o MAJ:MIN -n /dev/md0 | grep -q "254:" || restart=1 + if [ $restart = 1 ]; then + ${mdadm}/bin/mdadm --stop /dev/md0 + ${mdadm}/bin/mdadm --assemble /dev/md0 + sleep 1 + fi + ''; + }; + + cluster = { + k8sNode = true; + slurm = false; + }; + + features.hpc.slurm.mungeUid = 996; + + features = { + host = { + inherit address; + inherit name; + }; + + os = { + externalInterface = "eno1"; + nfs.enable = true; + nfs.exports = '' + /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + k8s = { + node.enable = true; + master.enable = false; + }; + }; + + systemd.services.rc-local = { + description = "rc.local script"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' + # if [ -e /sys/block/md126 ]; then + # echo "deadline" > /sys/block/md126/queue/scheduler + # # echo "4096" > /sys/block/md126/queue/nr_requests + # echo "4096" > /sys/block/md126/queue/read_ahead_kb + # echo "always" > /sys/kernel/mm/transparent_hugepage/enabled + # echo "always" > /sys/kernel/mm/transparent_hugepage/defrag + # fi + grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist + ''; + }; + + boot.kernel.sysctl = { + "vm.dirty_background_ratio" = 5; + "vm.dirty_ratio" = 10; + "vm.vfs_cache_pressure" = 50; + "vm.min_free_kbytes" = 262144; + }; + + networking = { + hostName = name; + interfaces.eno1 = { + useDHCP = false; + ipv4.addresses = [ { + address = address; + prefixLength = 24; + } ]; + ipv4.routes = [ + { + address = "10.255.242.0"; + prefixLength = 24; + via = "10.255.241.100"; + } + { + address = "10.1.8.0"; + prefixLength = 24; + via = "10.255.241.100"; + } + { + address = "10.1.30.0"; + prefixLength = 24; + via = "10.255.241.100"; + } + ]; + }; + interfaces."ibp59s0" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.81"; + prefixLength = 24; + } ]; + }; + interfaces."ibp59s1" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.82"; + prefixLength = 24; + } ]; + }; + firewall = { + allowedTCPPorts = []; + allowedUDPPorts = []; + extraCommands = '' + iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT + iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; + }; + + services.rpcbind.enable = true; + + systemd.mounts = [ + { + type = "nfs"; + what = "10.255.243.80:/data"; + where = "/data"; + mountConfig = { + Options = "soft,rdma,defaults,vers=4.2"; + }; + } + { + type = "nfs"; + what = "10.255.243.80:/opt"; + where = "/opt"; + mountConfig = { + Options = "soft,rdma,defaults,vers=4.2"; + }; + } + { + type = "nfs"; + what = "10.255.243.90:/work"; + where = "/work"; + mountConfig = { + Options = "soft,rdma,defaults,vers=4.2"; + }; + } + ]; + + systemd.automounts = [ + { + wantedBy = [ "multi-user.target" ]; + automountConfig = { + TimeoutIdleSec = "600"; + }; + where = "/data"; + } + { + wantedBy = [ "multi-user.target" ]; + automountConfig = { + TimeoutIdleSec = "600"; + }; + where = "/opt"; + } + ]; + + fileSystems = { + "/exports/backup" = { + device = "/backup"; + options = [ "bind" ]; + }; + }; + + programs.singularity.enable = true; + + imports = [ + ../cluster.nix + ./hardware-configuration.nix + ]; + }; + +} diff --git a/cluster/fs2/fs2.pub b/cluster/fs2/fs2.pub new file mode 100644 index 0000000..cf287ba --- /dev/null +++ b/cluster/fs2/fs2.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCb3/Z4YEFmeeaqQYzATkHgNaDWaT6MjiQsrIFYlZsvdXyE5QjGZMeRBMgJQyoyeCm9ig3sMbY8qiWLc/Kg+rFnEy+4xO7GYzCfqo1YsETiD0K+m/TbOsz9M2m+N8k+REWpKp9njTOYwsjWbrQ6FQPeQaKmig7P0PXJsSS/SAUr0Ns98QtBA9KtfRbDDCUlG4T/+kfu4Y970u0ABOF+zrE2mzfuaSn7n/jJAcAusIUQ9iiTrI6WHAok8lg9jC/+piW6ToEN7/ZX/RDggj5U5ZeN2IsjD1TZq57b/v7BAVx2CA38IYjuYizJFIlL8o7hEP22an1i44crsaR8pwJcfeMWzQxVIm9TPhSZZB9Ibq1R1syNb9vH/svmR8csWl0Uemk3gUGBJwzltmuXhLBg8mobypfBThAI/ZQKslvM9G9r4LoeLmxOOUTbCfXhi3ndIkEC2vGvo74h58BQj/DvMJ3a+V6gTha4837e2LK3Fm/8XWtU42jNbNlmblhLZby7uKWWb6orCnxscRs3cjP3BObSh6h9DO2aXMTnGYSGFdUKrRLFRmzFr+MPdhyORqp+GQgtHkzFG9W3/otQHdDCcD2OeFOxwEr3jOwUgtGIO1ZmJFDB9EIVHWyRk519foeZq/EHcSmIxgbwTgkpi/w8NQ1j++2nDxQOUeWo3U8xiQhONQ== root@fs2 diff --git a/cluster/fs2/hardware-configuration.nix b/cluster/fs2/hardware-configuration.nix new file mode 100644 index 0000000..9cb6214 --- /dev/null +++ b/cluster/fs2/hardware-configuration.nix @@ -0,0 +1,41 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "mpt3sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/19b7e607-b138-442a-9026-3ae1092046c9"; + fsType = "ext4"; + }; + + fileSystems."/backup" = + { device = "/dev/vg1/data"; + fsType = "xfs"; + options = [ "noauto" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.eno2.useDHCP = lib.mkDefault true; + # networking.interfaces.eno3.useDHCP = lib.mkDefault true; + # networking.interfaces.eno4.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/cluster/hosts.nix b/cluster/hosts.nix index f7ae590..88d35b5 100644 --- a/cluster/hosts.nix +++ b/cluster/hosts.nix @@ -1,10 +1,18 @@ '' + 10.255.242.2 ekman-gw ekman-gw.compute.local ekman-gw.cluster.local + 10.255.242.3 front-gw front-gw.compute.local front-gw.cluster.local + 10.255.241.80 nfs0 nfs0.compute.local 10.255.241.90 nfs1 nfs1.compute.local + 10.255.241.80 fs0 fs0.compute.local + 10.255.241.90 fs1 fs1.compute.local + 10.255.241.81 fs2 fs2.compute.local 10.255.241.100 ekman ekman.compute.local ekman.cluster.local 10.255.241.100 etcd0 etcd0.compute.local 10.255.241.80 etcd1 etcd1.compute.local 10.255.241.90 etcd2 etcd2.compute.local + 10.255.241.99 frontend frontend.compute.local frontend.cluster.local + 10.255.243.99 ibfrontend ibfrontend.compute.local ibfrontend.cluster.local 10.255.241.101 c0-1 c0-1.compute.local 10.255.241.102 c0-2 c0-2.compute.local @@ -16,19 +24,35 @@ 10.255.241.108 c0-8 c0-8.compute.local 10.255.241.109 c0-9 c0-9.compute.local 10.255.241.110 c0-10 c0-10.compute.local - 10.255.241.111 c0-11 c0-1.compute.local + 10.255.241.111 c0-11 c0-11.compute.local 10.255.241.112 c0-12 c0-12.compute.local 10.255.241.113 c0-13 c0-13.compute.local 10.255.241.114 c0-14 c0-14.compute.local 10.255.241.115 c0-15 c0-15.compute.local 10.255.241.116 c0-16 c0-16.compute.local + 10.255.241.117 c0-17 c0-17.compute.local + 10.255.241.118 c0-18 c0-18.compute.local + 10.255.241.119 c0-19 c0-19.compute.local + + 10.255.241.121 c1-1 c1-1.compute.local + 10.255.241.122 c1-2 c1-2.compute.local + 10.255.241.123 c1-3 c1-3.compute.local + 10.255.241.124 c1-4 c1-4.compute.local + 10.255.241.125 c1-5 c1-5.compute.local + 10.255.241.126 c1-6 c1-6.compute.local + 10.255.241.127 c1-7 c1-7.compute.local + 10.255.241.128 c1-8 c1-8.compute.local 10.255.243.80 ibnfs0 ibnfs0.compute.local 10.255.243.90 ibnfs1 ibnfs1.compute.local + 10.255.243.80 ibfs0 ibfs0.compute.local + 10.255.243.90 ibfs1 ibfs1.compute.local + 10.255.243.81 ibfs2 ibfs2.compute.local 10.255.243.100 ibekman ibekman.compute.local 10.255.243.100 ibetcd0 ibetcd0.compute.local 10.255.243.80 ibetcd1 ibetcd1.compute.local 10.255.243.90 ibetcd2 ibetcd2.compute.local + 10.255.243.101 ib0-1 ib0-1.compute.local 10.255.243.102 ib0-2 ib0-2.compute.local 10.255.243.103 ib0-3 ib0-3.compute.local @@ -45,4 +69,16 @@ 10.255.243.114 ib0-14 ib0-14.compute.local 10.255.243.115 ib0-15 ib0-15.compute.local 10.255.243.116 ib0-16 ib0-16.compute.local + 10.255.243.117 ib0-17 ib0-17.compute.local + 10.255.243.118 ib0-18 ib0-18.compute.local + 10.255.243.118 ib0-18 ib0-19.compute.local + + 10.255.243.121 ib1-1 ib1-1.compute.local + 10.255.243.122 ib1-2 ib1-2.compute.local + 10.255.243.123 ib1-3 ib1-3.compute.local + 10.255.243.124 ib1-4 ib1-4.compute.local + 10.255.243.125 ib1-5 ib1-5.compute.local + 10.255.243.126 ib1-6 ib1-6.compute.local + 10.255.243.127 ib1-7 ib1-7.compute.local + 10.255.243.128 ib1-8 ib1-8.compute.local '' diff --git a/cluster/compute/munge.key b/cluster/munge.key similarity index 100% rename from cluster/compute/munge.key rename to cluster/munge.key diff --git a/cluster/overlays.nix b/cluster/overlays.nix index ea76aba..87e6e6f 100644 --- a/cluster/overlays.nix +++ b/cluster/overlays.nix @@ -4,8 +4,28 @@ let configureFlags = attrs.configureFlags ++ [ "--with-tls=openssl" ]; buildInputs = attrs.buildInputs ++ [ super.openssl ]; }); + + lib = super.lib; + squashfsTools = super.squashfsTools; + cryptsetup = super.cryptsetup; + + singularity = super.singularity.overrideAttrs (attrs: rec { + installPhase = '' + runHook preInstall + make -C builddir install LOCALSTATEDIR=$out/var + chmod 755 $out/libexec/singularity/bin/starter-suid + + # Explicitly configure paths in the config file + sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf + sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf + sed -i 's|^shared loop devices = no|shared loop devices = yes|' $out/etc/singularity/singularity.conf + + runHook postInstall + ''; + }); in { + inherit singularity; # inherit msmtp; } diff --git a/cluster/users.nix b/cluster/users.nix index a89775c..ebf4bfe 100644 --- a/cluster/users.nix +++ b/cluster/users.nix @@ -7,7 +7,26 @@ bast = { gid = 1003; }; stig = { gid = 1004; }; simenlk = { gid = 1005; }; - serf = { gid = 2001; }; + kaih = { gid = 1006; }; + + hipster = { + members = [ + "jonas" + "olean" + "frankgaa" + "stig" + ]; + }; + + data = { + gid = 2002; + members = [ + "jonas" + "olean" + "frankgaa" + "stig" + ]; + }; sif = { gid = 11000; @@ -20,22 +39,19 @@ "simenlk" ]; }; + + kraken = { + gid = 5000; + members = [ + "jonas" + "olean" + "frankgaa" + "stig" + ]; + }; }; users.users = { - serf = { - description = "Oceanbox job runner"; - home = "/work/serf"; - group = "serf"; - extraGroups = [ - "users" - ]; - uid = 2001; - isNormalUser = true; - createHome = false; - useDefaultShell = true; - }; - jonas = { description = "Jonas Juselius"; home = "/home/jonas"; @@ -62,6 +78,7 @@ openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas-3" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIJSAEsqKbQg2ZK5pKL0wbe+hzkDjsWzPb0BgT/7vLu/KAAAADHNzaDpvY2VhbmJveA== jonas@foldr" ]; }; @@ -140,7 +157,6 @@ openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHVwcJOtx9YTWy+aD4xGbyPFLOdMN6NqY8wcfDtHczyT Stig Rune Jensen" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfgY468dPNpdXZCkD9jw1p2qA0+z56Wi/c1VYE+riki Stig Rune Jensen" - ]; }; @@ -158,8 +174,64 @@ shell = pkgs.fish; openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCzchzNR/oVJ6OgeEAmDI1jT5WTgJRk+w2XR0Euh/S+9vQ5Zlj97zRsul9KBhJLaVS2OxVl/v7gc0Cm2oyXcIacfoIF3fw4SPy/T40AHvsD6cR5H0C+q0cCFA1rroK9g4hybyc0GK+A2xs21CMfmN1C7V6jtKRz+6DKUDBcbCL+YgGo+rXxzmUPvyy9mmvx7Ae1/GZ1WoAiCWVCgE+dSmuiOtsEYVqX8eqBb73rk0wf+ijnSFSOPsVpJZciPBQ3gmyAttl1KIAoQtMxK2kFDu9i71LjnU+e4xyaJ2RuJ5747QWSne45VMShL08H56IySoY2SPboq/sOL7lg/yo9yjOCsUMjwXwzUkwvmZZX0iW5Y/yEr8WPXFI4u9rZEmAeHHt7ky+UkT3ZbXk8kcOJdALmB15iFPbtWS+7Ctrn+5R94988O874j8MNAyeZaGAeoBD75pb/EsjR6ohEV6XwBl5TQ32wexwd1sV0fTVjMFYKZGqXs8oX20o4vrbM964hG7k= adminbrede@DESKTOP-QAOIKJD" + ]; + }; + + kaih = { + description = "Kai Hanssen"; + home = "/home/kaih"; + group = "kaih"; + extraGroups = [ + "users" + ]; + uid = 1006; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5k0dXn60dZ3iORy99LVvgTldu9nYU1TJVL1wCJEqp kaih kubernetes" + ]; + }; + + # achim = { + # description = "Achim Randelhoff"; + # home = "/tmp/achim"; + # group = "users"; + # extraGroups = []; + # uid = 1016; + # isNormalUser = true; + # createHome = false; + # useDefaultShell = false; + # shell = pkgs.fish; + # openssh.authorizedKeys.keys = [ + # ]; + # }; + + kraken = { + description = "The Kraken"; + home = "/work/kraken"; + group = "kraken"; + extraGroups = [ "users" ]; + uid = 5000; + isNormalUser = true; + createHome = true; + useDefaultShell = false; + shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIJSAEsqKbQg2ZK5pKL0wbe+hzkDjsWzPb0BgT/7vLu/KAAAADHNzaDpvY2VhbmJveA== jonas@foldr" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCp3QEG9uJq4EKMRWlBkqg/EQD6+2E90E6/1i+JLbiBYBSV6Yqac6KzyJezaVXzch+qm6FFXas3thaBScbA4cELkETaxyNzYG6sNcLyEhlQn51pEsq2mFiq8IiaMaDc55/2HTbXVrpNoTNQFt6lBwHziKQYuUI0H0cxze+ppp0ZJOu1MsayW8JOv75YSv6WFIDRR+KP6dOEBu1PsP6plTZwK94ogjQ+3KHGcMAnE3cf8VCEF8akNC6GRmCgXNZE4I9MmKTDr217OoFpnXAx5/KTvGo+USkXc6xn/vbQsni4ExwGlMTg97RK49wIHD1NfGxZ3sv7mZ+UQPqqmSxCG+zueJrR6BSBfbm7fw5KvRn69rOihapeo/6GoqqVDe4yn1imtojjHN6+9pgJ9E6o108qbXRw2X6t1KUuXrB+fTfUKvy0kWiJFIMDSUtKF/nhiES+aCI4b4WBwyg5hdKGvgJdjyUS7P/jYgqWRe+qmknAERtQKlFDA/C6ChsTXerFD5Ikvu3dajJUiDehszEON5F4JlxSf2VpUFCDLVNqV/GjJqOg90mXGDk82c+0ZHIUsPLsdqR+t/xnOSv1Ks9I5fId6g+3OlR1ifnb3Qm48QGKbi/CM8M/QXzv4VgeIkRTR0Oi4W0P1tpUSyPd1nyGaM/B/FqN52XIUjRqIfu0emwgiw== Ole Anders Nøst" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDyOEungdZOZRREq3B1STmlQm91a/R8E06dcq5xni6+N8EBU3RJuJ2zIW/FVaID/TrkLesomIl0/X6K3dM1tpMdaV5IemCt7/8TEkVr+dEq/vDEPeeDvBa2hpoQNzdMXQTCbHgP2O2YC7JovKXO2saGEsvzDDcI0hJoTjZ/gqBAkwTMZy/uSStR7045L/f901To6tiahnoIH9m3YtIkn1wluco0/3Zr3xFlDAAl6P3sN8n/vnPqUYE1GE8e6QdmC9n9Zc8vvN9YtmJSju0BC1sZvY+pPI+aS9a4Y7OUGj7jLRVdGX+adW1SQJ6eHO6DMHt1ua8nOq2MDazIQEmnfjZuhGckmu+cGV8n9cvIKmIRT+/RCBmpD2hadzJl9OGvLuvRynlvmeEbT6y/I7YZhwKAOm0qp1tLS2fVD3bj328atlFefu4o1enJbjQRH6PAWOxC+b3war1hzkNIsmViaiO7Fd1dHV73y1OJR863+Ry6uQW2hQvtNiG79p4EhRTSQQs= Frank Gaardsted" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXv//iSTq5k1iYPZq9MKuF2OCE2UQPdR6TglWAeAnwL72UoKG068qDK3Ys+mFRY/S957hLn0FPFJPigcSaLzBq2v3G6kKySg68XWQfxBTSi5Wg81taNoJKQK+QAKfU2FiW9i1dLkvRKEXlY3tmF2mZLwqvrClRQMw8Nz0PQ5LnQfRgge07aXA3nDEf/nRuIaPG8zEki56lOONMWz4bGTjPn1y6y9gYAmskOc9w7uEOAy3VxkoR8fKvQM0ZTgt+6+68QxReyaDHH+12AqDxDy4wTCNX1LU902NDxyJZrUa2Xv9me2qN5O7hDOL/8S19MkJPBDEttMtA2rsFQlD9WYqycqgFhbOzLb7gixK1lrL3CYHsE9fXD2LuitSDXf79HFnCVHD5HJG7CbpIJLNNeTOCx94vspf9J8OENNdnNCgMFC1FKV/vdCiZ/RAOUCINrekvrX8FNjlXIhHOeK/gG6gP61oWpx3qbOExeMQTqWa9cGeHPtIdPiJVCza9Mg4X+0D9DCaP7KVLAxKioWqyd2WsyYeVhXA0OqnkEQk/jPZUjnxL1rnlH3I6QtVxHyqKcmmWEoRUnXId0ASUqx2hmsmI0TZD197PLFq53VO86v7jlAXLyzwmPh5VWdTOywklRpM29sZplVG/6gkHm/vM0DVSTEjVW+4mMFlzaiKgujH2fw== Frank Gaardsted" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHVwcJOtx9YTWy+aD4xGbyPFLOdMN6NqY8wcfDtHczyT Stig Rune Jensen" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfgY468dPNpdXZCkD9jw1p2qA0+z56Wi/c1VYE+riki Stig Rune Jensen" ]; }; }; + security.sudo.extraConfig = '' + %kraken ALL=(kraken) NOPASSWD: ALL + ''; } diff --git a/configuration.nix b/configuration.nix index 6e138ee..84bc43a 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,9 +1,22 @@ { pkgs, ...}: let - computeNodes = import ./cluster/compute/nodes.nix; + computeNodes = + import ./cluster/c0/nodes.nix ++ + import ./cluster/c1/nodes.nix ++ + [ + rec { + idx = 100; + name = "ekman"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ./cluster/ekman/ekman.pub; + } + ]; etcdCluster = import ./cluster/etcdCluster.nix; -in -{ + name = "frontend"; + address = "10.255.241.99"; + ipoib = "10.255.243.99"; +in { systemd.targets = { sleep.enable = false; suspend.enable = false; @@ -11,30 +24,41 @@ in hybrid-sleep.enable = false; }; + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + ]; + cluster = { - compute = true; - k8sNode = true; + k8sNode = true; + compute = false; + slurm = true; }; features = { host = { - address = "10.255.241.100"; - name = "ekman"; + inherit address; + inherit name; }; - myvnc.enable = true; + myvnc.enable = false; os = { - externalInterface = "enp33s0f0np0"; - nfs.enable = true; + externalInterface = "eno1"; + nfs.enable = false; nfs.exports = '' - /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) ''; }; hpc = { slurm.server = true; + slurm.mungeUid = 996; frontend = true; }; @@ -48,7 +72,13 @@ in monitoring = { server = { enable = false; - scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); + scrapeHosts = [ + "frontend" + "ekman" + "fs0" + "fs1" + "fs2" + ] ++ (builtins.map (x: x.name) computeNodes); defaultAlertReceiver = { email_configs = [ { to = "jonas.juselius@oceanbox.io"; } @@ -75,7 +105,11 @@ in infiniband-exporter = { enable = true; nameMap = '' - 0xe8ebd3030024a2c6 "frontend" + 0xe8ebd3030024a2c6 "ekman" + 0x0c42a10300ddc4bc "frontend" + 0xe8ebd3030024a2a2 "fs0" + 0xe8ebd3030024a2ae "fs1" + 0x1c34da0300787798 "fs3" 0xe8ebd3030024981e "c0-1" 0xe8ebd3030024a21a "c0-2" 0xe8ebd30300249a3a "c0-3" @@ -92,8 +126,14 @@ in 0xe8ebd303002495e2 "c0-14" 0xe8ebd30300248f42 "c0-15" 0xe8ebd303002495e6 "c0-16" - 0xe8ebd3030024a2a2 "nfs0" - 0xe8ebd3030024a2ae "nfs1" + 0x0c42a10300dbe7f4 "c1-1" + 0x0c42a10300dbe7d8 "c1-2" + 0x0c42a10300dbe800 "c1-3" + 0x0c42a10300dbec80 "c1-4" + 0x0c42a10300dbea50 "c1-5" + 0x0c42a10300dbeb2c "c1-6" + 0x0c42a10300dbe7fc "c1-7" + 0x0c42a10300dbe5a0 "c1-8" ''; }; slurm-exporter = { @@ -103,17 +143,17 @@ in }; }; - services.udev.extraRules = '' - KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" - ''; + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; networking = { useDHCP = false; - hostName = "ekman"; - interfaces.enp33s0f0np0 = { + hostName = name; + interfaces.eno1 = { useDHCP = false; ipv4.addresses = [ { - address = "10.255.242.2"; + address = "10.255.242.3"; prefixLength = 24; } ]; ipv4.routes = [ @@ -129,22 +169,23 @@ in } ]; }; - interfaces.enp33s0f3np3 = { + interfaces.enp175s0f0 = { useDHCP = false; - ipv4.addresses = [ { - address = "10.255.241.100"; - prefixLength = 24; - } ]; + ipv4.addresses = [ + { + inherit address; + prefixLength = 24; + } + ]; }; - interfaces."ibp65s0" = { + interfaces.ibp59s0 = { useDHCP = false; - }; - interfaces."ibp65s0.7666" = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.255.243.100"; - prefixLength = 24; - } ]; + ipv4.addresses = [ + { + address = ipoib; + prefixLength = 24; + } + ]; }; defaultGateway = "10.255.242.1"; firewall = { @@ -152,37 +193,41 @@ in extraCommands = '' iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT - iptables -t nat -A POSTROUTING -s 10.255.241.0/24 -j SNAT --to-source 10.255.242.2 + # needed for nodeport access on k1 and k2 + iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.241.0/24 -j SNAT --to-source 10.255.242.3 iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE ''; }; }; fileSystems = { - "/exports/home" = { - device = "/home"; - options = [ "bind" ]; + "/opt" = { + device = "10.255.243.80:/opt"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; }; - "/frontend" = { - device = "/home"; - options = [ "bind" ]; + "/data" = { + device = "10.255.243.80:/data"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; }; - "/vol/local-storage/vol1" = { - device = "/vol/vol1"; - options = [ "bind" ]; + "/work" = { + device = "10.255.243.90:/work"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" "vers=4.2" ]; }; - "/vol/local-storage/vol2" = { - device = "/vol/vol2"; + "/exports/public" = { + device = "/srv/public"; options = [ "bind" ]; }; }; nix.extraOptions = '' - secret-key-files = /etc/nix/ekman.key + # secret-key-files = /etc/nix/ekman.key ''; services.xserver = { - enable = true; + enable = false; enableCtrlAltBackspace = true; layout = "us"; xkbVariant = "altgr-intl"; @@ -202,64 +247,6 @@ in smtp_from = "noreply@ekman.oceanbox.io"; }; - systemd.sockets.slurm-http-proxy = { - enable = true; - description = "Proxy slurmrestd unix socet to port 6822"; - listenStreams = [ "0.0.0.0:6822" ]; - wantedBy = [ "sockets.target" ]; - # Allow multiple instances of corresponding service. - socketConfig.Accept = true; - }; - - systemd.services."slurm-http-proxy@" = { - enable = true; - description = "Proxy slurmrestd unix socet to port 6822"; - serviceConfig = { - # ExecStart = "-${pkgs.socat}/bin/socat -d -d -d TCP4-LISTEN:6822,fork,reuseaddr UNIX-CONNECT:/run/slurmrestd/serf.socket"; - ExecStart = "-${pkgs.socat}/bin/socat -d -d -d STDIO UNIX-CONNECT:/run/slurmrestd/serf.socket"; - StandardInput="socket"; - }; - }; - - # services.nginx = { - # virtualHosts = { - # "ds.matnoc.regnekraft.io" = { - # forceSSL = true; - # enableACME = true; - # serverAliases = []; - # locations."/" = { - # proxyPass = "http://localhost:9088"; - # proxyWebsockets = false; - # extraConfig = '' - # allow 10.1.2.0/24; - # allow 172.19.254.0/24; - # allow 172.19.255.0/24; - # deny all; - # ''; - # }; - # }; - # }; - # }; - - # services.gitlab-runner = { - # enable = true; - # extraPackages = with pkgs; [ - # singularity - # ]; - # concurrent = 4; - # services = { - # sif = { - # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; - # executor = "shell"; - # tagList = [ "ekman" "sif" ]; - # }; - # }; - # }; - - # security.sudo.extraConfig = '' - # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity - # ''; - security.pam = { services.sshd.googleAuthenticator.enable = true; loginLimits = [ @@ -279,23 +266,23 @@ in }; system.activationScripts = { - home-permissions.text = '' - chmod 755 /home/olean - chmod 755 /home/frankgaa - chmod 755 /home/jonas - chmod 755 /home/stig - chmod 755 /home/bast - chmod 755 /home/simenlk - ''; + # home-permissions.text = '' + # chmod 755 /home/olean + # chmod 755 /home/frankgaa + # chmod 755 /home/jonas + # chmod 755 /home/stig + # chmod 755 /home/bast + # chmod 755 /home/simenlk + # chmod 755 /work/kraken + # ''; }; # ssh-rsa is deprecated, but putty/winscp users use it - # services.openssh.extraConfig = '' - # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 - # ''; - - environment.systemPackages = []; + services.openssh.extraConfig = '' + # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 + PubkeyAuthOptions verify-required + ''; imports = [ ./cluster/cluster.nix diff --git a/frontend.pub b/frontend.pub new file mode 100644 index 0000000..b4c0d9c --- /dev/null +++ b/frontend.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkSSuHkieXwgFMKRy4MjwjNrJEItWbQHeAAH+Zn1YZp root@localhost diff --git a/hardware-configuration.nix b/hardware-configuration.nix index 57814eb..84acc26 100644 --- a/hardware-configuration.nix +++ b/hardware-configuration.nix @@ -8,32 +8,20 @@ [ (modulesPath + "/installer/scan/not-detected.nix") ]; - boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; boot.initrd.kernelModules = [ ]; - boot.kernelModules = [ "kvm-amd" ]; + boot.kernelModules = [ "kvm-intel" ]; boot.extraModulePackages = [ ]; fileSystems."/" = - { device = "/dev/disk/by-uuid/c4a5ea80-2a87-440d-8431-a695d623da24"; + { device = "/dev/disk/by-uuid/099be829-726d-44c9-b113-4c2604a663c5"; fsType = "ext4"; }; fileSystems."/boot" = - { device = "/dev/disk/by-uuid/D8A3-A04D"; + { device = "/dev/disk/by-uuid/8111-9F24"; fsType = "vfat"; }; swapDevices = [ ]; - - # Enables DHCP on each ethernet and wireless interface. In case of scripted networking - # (the default) this is the recommended approach. When using systemd-networkd it's - # still possible to use this option, but it's recommended to use it in conjunction - # with explicit per-interface declarations with `networking.interfaces..useDHCP`. - networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true; - # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true; - - hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; } diff --git a/hive.nix b/hive.nix index 6d8d36f..de9f362 100644 --- a/hive.nix +++ b/hive.nix @@ -6,17 +6,20 @@ let # }) {}; pkgs = import {}; - ekman = { + frontend = { deployment = { - tags = [ "ekman" "frontend" ]; + tags = [ "manage" "frontend" ]; allowLocalDeployment = true; targetHost = null; }; imports = [ ./configuration.nix ]; }; - compute = import ./cluster/compute { inherit pkgs; }; - nfs0 = import ./cluster/nfs0 { inherit pkgs; }; - nfs1 = import ./cluster/nfs1 { inherit pkgs; }; + ekman = import ./cluster/ekman { inherit pkgs; }; + c0 = import ./cluster/c0 { inherit pkgs; }; + c1 = import ./cluster/c1 { inherit pkgs; }; + fs0 = import ./cluster/fs0 { inherit pkgs; }; + fs1 = import ./cluster/fs1 { inherit pkgs; }; + fs2 = import ./cluster/fs2 { inherit pkgs; }; in - { inherit ekman; } // compute // nfs0 // nfs1 + { inherit frontend; stokes = frontend; } // ekman // c0 // c1 // fs0 // fs1 // fs2 diff --git a/modules b/modules index 658556e..3c35d46 160000 --- a/modules +++ b/modules @@ -1 +1 @@ -Subproject commit 658556eb6ef854d2c7e7d9320210851bd457ee4b +Subproject commit 3c35d466b071c97af4646b1273c3d373f6d1c421 diff --git a/nixos b/nixos index b1b335c..a250a69 160000 --- a/nixos +++ b/nixos @@ -1 +1 @@ -Subproject commit b1b335c936cb33f648c070f4d758bfc7414698a0 +Subproject commit a250a69aafd6da6a9c208e6a1f911caf7bf2bf55