major: initial rossy cluster and biggish refactor

rossby/c0/connauthfile

@@ -0,0 +1 @@
+莵q丘

rossby/c0/default.nix

@@ -0,0 +1,106 @@
+{ pkgs ? import <nixpkgs> {} }:
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
+  # }) {};
+  # pkgs = import <nixpkgs> {};
+
+  nodes = import ./nodes.nix;
+
+  compute = {
+    deployment.tags = [ "compute" "c0" ];
+
+    fileSystems = {
+      "/users" = {
+        device = "172.16.239.222:/home";
+        fsType = "nfs4";
+        options = [
+          "soft"
+          "defaults"
+          "noauto"
+          "x-systemd.automount"
+        ];
+      };
+    };
+
+    systemd.automounts = [
+      {
+        where = "/users";
+        wantedBy = [ "default.target" ];
+      }
+    ];
+  };
+
+  mkCompute = host:
+  let
+    hw = ./hardware-config.d + "/${host.name}.nix";
+  in {
+    "${host.name}" = {
+      cluster = {
+        compute = true;
+        k8sNode = true;
+        mounts = {
+          rdma.enable = false;
+          automount.enable = true;
+          users = true;
+          opt = true;
+          work = false;
+          data = false;
+          ceph = false;
+        };
+      };
+
+      features = {
+        host = {
+          name = host.name;
+          address = host.address;
+        };
+        os.externalInterface = "enp65s0f0";
+        hpc.compute = true;
+        # hpc.knem = true;
+      };
+
+      deployment.targetHost = host.address;
+
+      # services.udev.extraRules = ''
+      #   KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+      # '';
+
+      networking = {
+        hostName = host.name;
+        useDHCP = false;
+        interfaces. enp65s0f0 = {
+          useDHCP = false;
+          ipv4.addresses = [ {
+            address = host.address;
+            prefixLength = 24;
+          } ];
+          # ipv4.routes = [ {
+          #   address = "10.255.242.0";
+          #   prefixLength = 24;
+          #   via = "10.255.241.100";
+          # } ];
+
+        };
+        # interfaces."ibp1s0.7666" = {
+        # interfaces."ibp1s0" = {
+        #   useDHCP = false;
+        #   ipv4.addresses = [ {
+        #     address = host.ipoib;
+        #     prefixLength = 24;
+        #   } ];
+        # };
+      };
+      imports = [
+          hw
+          ../default.nix
+          ../mounts.nix
+          #./kernel.nix
+      ];
+    }
+    // compute;
+};
+in builtins.foldl' (a: n: a // mkCompute n) {} nodes
+

rossby/c0/hardware-config.d/c0-10.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/9e236fcb-00e5-43a7-ad4b-41b1b068cd5d";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/28C7-DCBE";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-11.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/92a043ce-b2be-4cdd-b6e0-1e8f3e6952fd";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3038-56D0";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-13.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/f0909558-f71a-472e-af38-8ffdb2e9e453";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/72A0-CF21";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-14.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/c35b6b6f-29da-4ea0-8fba-067301ce9572";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3FC3-84E2";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-15.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/1a765ce5-2156-4427-8594-824473a31205";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/4CD7-6D3E";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-16.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/2580dfeb-a28e-4f3d-87a8-e7c7a9fdfeca";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/6BB5-3DC2";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-17.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/5d8be7db-bdb6-47d2-bb9a-5dc62f162c0f";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/8C14-D452";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-18.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/0a0feff4-c3aa-4a67-9880-78c08f2e1db3";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/A52D-B7FA";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-19.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/cd6d1033-a92f-4ab7-8265-00d57466221b";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/AEA2-6A41";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-2.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/032fe2e6-9091-47e8-b121-0123abe4f0fc";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/EE11-AD60";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-20.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/7dab9648-6618-46df-978e-f0ac43405fd5";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/08D3-CB99";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-3.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/da170cd8-0a6b-470f-96aa-a7bdf3dd8736";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/159D-E4C1";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-4.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/b081d8a1-3cae-4017-afd3-0482d58f3062";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/1E63-EA5E";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-5.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/494bceb1-c0d1-419d-a379-8104ff90aa61";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/294F-20BD";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-6.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/be0a7204-1a90-4f06-980a-a315e9c664ca";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/4889-6E82";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-8.nix

@@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/ffeb8ede-16f1-48bc-a976-1ea9ab982745";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/E780-1C40";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/hardware-config.d/c0-9.nix

@@ -0,0 +1,44 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/3d27bbdb-4370-4af2-9747-74d03acfa558";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/9E1B-2495";
+      fsType = "vfat";
+      options = [ "fmask=0077" "dmask=0077" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
+  # networking.interfaces.tailscale0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

rossby/c0/kernel.nix

@@ -0,0 +1,58 @@
+{pkgs, lib, ...}:
+let
+  kernel = pkgs.linuxPackages.kernel;
+  i40e =
+    pkgs.stdenv.mkDerivation rec {
+      name = "i40e-${version}-${kernel.version}";
+      version = "2.13.10";
+
+      src = pkgs.fetchFromGitHub {
+        owner = "dmarion";
+        repo = "i40e";
+        rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
+        sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
+      };
+
+      hardeningDisable = [ "pic" ];
+
+      configurePhase = ''
+        cd src
+        kernel_version=${kernel.modDirVersion}
+        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
+        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
+        export makeFlags="BUILD_KERNEL=$kernel_version"
+      '';
+
+      installPhase = ''
+        install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
+      '';
+
+      dontStrip = true;
+
+      enableParallelBuilding = true;
+
+      meta = {
+        description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
+        homepage = https://github.com/dmarion/i40e;
+        license = lib.licenses.gpl2;
+      };
+    };
+in
+{
+  # i40e2 = i40e;
+  # boot.kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override {
+  #   argsOverride = rec {
+  #     src = pkgs.fetchurl {
+  #           url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
+  #           sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid";
+  #     };
+  #     version = "5.10.239";
+  #     modDirVersion = "5.10.239";
+  #     };
+  # });
+  boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
+  # overlay = self: super: {
+    # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
+  # };
+}
+

rossby/c0/nodes.nix

@@ -0,0 +1,11 @@
+with builtins;
+let
+  nodes = genList (n: n + 1) 20; in
+map (n: (
+  rec {
+    idx = 110 + n;
+    name = "c0-${toString n}";
+    address = "172.16.239.${toString idx}";
+    ipoib = "172.16.240.${toString idx}";
+    pubkey = ./. +  "/ssh_host_key.d/c0-${toString n}.pub";
+  })) (filter (x: if x == 1 || x == 7 || x == 12 then false else true) nodes)

rossby/c0/ssh_host_key.d/c0-10.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBd59La09RhuP0kiYXSGqAIY9ZV9lKzMOuqXUCHfNuRA root@nixos

rossby/c0/ssh_host_key.d/c0-11.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcaGpXQqp/3WwI0iJlHuIuNvq60Pan8mj2Jri2J8V0W root@nixos

rossby/c0/ssh_host_key.d/c0-13.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICAEVDoSKJc41wkIOfsYcKezmQwLADtcU5WUGz7+gWP9 root@nixos

rossby/c0/ssh_host_key.d/c0-14.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdrfN8vG/UUjUj+JrXAnfS7wYmiO+mmuTEElfUDOSTA root@nixos

rossby/c0/ssh_host_key.d/c0-15.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHil3FeI6GxrpJ52XlZDr6FBQ0gdjX4WrnLH5bHmSYXz root@nixos

rossby/c0/ssh_host_key.d/c0-16.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGaHMHXeHdL/rOaVWPVPx7rpL6UWBiOaHKOaQdzOFc/3 root@nixos

rossby/c0/ssh_host_key.d/c0-17.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF+YvVsZwsFeyghA4hG6o0ja+zjN3d42p//8fwOuYyDQ root@nixos

rossby/c0/ssh_host_key.d/c0-18.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMkeYwO3dGG+l3tmTvP23xroT2Dm+aZb46ISIjzWvIkk root@nixos

rossby/c0/ssh_host_key.d/c0-19.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKTKLGyYiuRMUNqim4geRv6bramCigPdzA3Ed4Meupkd root@nixos

rossby/c0/ssh_host_key.d/c0-2.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBnQW4aNsmDrNu8yXa8xJtKNWi9Cf2BHZ7Bc/SP26zs root@nixos

rossby/c0/ssh_host_key.d/c0-20.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILELid8+ax9zHcm3Jbv5oE1XMVmqyjmR+/KyA+thd5v4 root@nixos

rossby/c0/ssh_host_key.d/c0-3.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHydtfeTvjqWDQnMPdLAWwirZvV6DrBe2K7yZ3lR1oQy root@nixos

rossby/c0/ssh_host_key.d/c0-4.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO+0beDz0Tk3wuBTk8miH/NKE4U/IbHPq/qMj+y644iV root@nixos

rossby/c0/ssh_host_key.d/c0-5.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC+23znDppuQCDRgpoovlGUG113nZ9vFKrBST8sf0BwP root@nixos

rossby/c0/ssh_host_key.d/c0-6.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJzUH3PJi8v0vQfzj2WXsi9IM/0MVpZ87Clu8LyCepZj root@nixos

rossby/c0/ssh_host_key.d/c0-8.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICecn4QgdQmK/tHB1Q5EhitExEuVm+JtnyB7eOqIwGrj root@nixos

rossby/c0/ssh_host_key.d/c0-9.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICf1lE0k49jPMxixAfAwqC2s3Wov8k3P8WgSQ6QTUHO9 root@rossby-manage