wip: convert ekman to new cluster sturcture (not complete)

ekman/c1/connauthfile

@@ -0,0 +1 @@
+莵q丘

ekman/c1/default.nix

@@ -0,0 +1,119 @@
+{ pkgs ? import <nixpkgs> {} }:
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
+  # }) {};
+  # pkgs = import <nixpkgs> {};
+
+  nodes = import ./nodes.nix;
+  compute = {
+    deployment.tags = [ "compute" "c1" ];
+
+    systemd.automounts = [
+      {
+        where = "/frontend";
+        wantedBy = [ "default.target" ];
+      }
+    ];
+  };
+
+  mkCompute = host:
+  let
+    hw = ./hardware-configuration.d + "/${host.name}.nix";
+  in {
+    "${host.name}" = {
+      cluster = {
+        compute = true;
+        k8sNode = true;
+        mounts = {
+          rdma.enable = false;
+          gbe100.enable = true;
+          automount.enable = true;
+          home = true;
+          opt = true;
+          work = true;
+          data = false;
+          backup = false;
+          ceph = true;
+        };
+      };
+
+      features = {
+        host = {
+          name = host.name;
+          address = host.address;
+        };
+        os.externalInterface = "eno33np0";
+        hpc.compute = true;
+        # k8s = { inherit etcdCluster; };
+      };
+
+      deployment.targetHost = host.target;
+
+      # services.udev.extraRules = ''
+      #   KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+      # '';
+
+      # boot.kernel.sysctl = {
+      #    "net.ipv4.tcp_timestamps" = 0;
+      #    "net.ipv4.tcp_sack" = 1;
+      #    "net.core.netdev_max_backlog" = 250000;
+      #    "net.core.rmem_max" = 4194304;
+      #    "net.core.wmem_max" = 4194304;
+      #    "net.core.rmem_default" = 4194304;
+      #    "net.core.wmem_default" = 4194304;
+      #    "net.core.optmem_max" = 4194304;
+      #    "net.ipv4.tcp_rmem" = "4096 87380 4194304";
+      #    "net.ipv4.tcp_wmem" = "4096 65536 4194304";
+      #    "net.ipv4.tcp_low_latency" = 1;
+      #    "net.ipv4.tcp_adv_win_scale" = 1;
+      # };
+
+      networking = {
+        hostName = host.name;
+        useDHCP = false;
+        interfaces.eno33np0 = {
+          useDHCP = false;
+          ipv4.addresses = [ {
+            address = host.address;
+            prefixLength = 24;
+          } ];
+          ipv4.routes = [ {
+            address = "10.255.242.0";
+            prefixLength = 24;
+            via = "10.255.241.100";
+          } ];
+
+        };
+        # interfaces.ibp65s0 = {
+        #   useDHCP = false;
+        #   ipv4.addresses = [ {
+        #     address = host.ipoib;
+        #     prefixLength = 24;
+        #   } ];
+        # };
+        interfaces.enp65s0np0 = {
+          useDHCP = false;
+          ipv4.addresses = [ {
+            address = host.gbe100;
+            prefixLength = 24;
+          } ];
+        };
+        # firewall.extraCommands =
+          # if host.name == "c1-1" then ''
+          #   iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE
+          # ''
+          # else "";
+      };
+      imports = [
+          hw
+          ../cluster.nix
+          ../mounts.nix
+      ];
+    }
+    // compute;
+};
+in builtins.foldl' (a: n: a // mkCompute n) {} nodes
+

ekman/c1/hardware-configuration.d/c1-1.nix

@@ -0,0 +1,29 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+  # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_7;
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/d89e1496-fda1-4de0-b2cc-474967b04402";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/A51A-1F4D";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-2.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/f0826ad5-8a4e-427d-98d3-5afa44440993";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/21A6-D34C";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-3.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/3af91585-8079-420d-acdf-f60b94d3cfff";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3590-199A";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-4.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/84cc9cea-08eb-4b54-8ca3-2aa5c1300a92";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/882D-A342";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-5.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/d23386a8-0ca8-4871-b662-decf2b24f4d7";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/AF8A-DEFE";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-6.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/2f468316-5832-4684-866d-2e92b08fb68b";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/76FE-F657";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-7.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/37944fce-07ca-492f-906a-620a37e7e1b3";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/2F51-EC20";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-8.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/12477966-c6c5-47c6-afdc-35fa7e57e837";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/DB96-7453";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/kernel.nix

@@ -0,0 +1,46 @@
+{pkgs, lib, stdenv, fetchurl, config, kernel ? pkgs.linux, ...}:
+let
+  i40e =
+    stdenv.mkDerivation rec {
+      name = "i40e-${version}-${kernel.version}";
+      version = "2.13.10";
+
+      src = pkgs.fetchFromGitHub {
+        owner = "dmarion";
+        repo = "i40e";
+        rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
+        sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
+      };
+
+      hardeningDisable = [ "pic" ];
+
+      configurePhase = ''
+        cd src
+        kernel_version=${kernel.modDirVersion}
+        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
+        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
+        export makeFlags="BUILD_KERNEL=$kernel_version"
+      '';
+
+      installPhase = ''
+        install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
+      '';
+
+      dontStrip = true;
+
+      enableParallelBuilding = true;
+
+      meta = {
+        description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
+        homepage = https://github.com/dmarion/i40e;
+        license = lib.licenses.gpl2;
+      };
+    };
+in
+{
+  i40e2 = i40e;
+  overlay = self: super: {
+    # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
+  };
+}
+

ekman/c1/nodes.nix

@@ -0,0 +1,12 @@
+with builtins;
+let nodes = genList (n: n + 1) 8; in
+map (n: (
+  rec {
+    idx = 120 + n;
+    name = "c1-${toString n}";
+    target = "10.255.241.${toString (idx + 100)}";
+    address = "10.255.241.${toString idx}";
+    ipoib = "10.255.243.${toString idx}";
+    gbe100 = "10.255.244.${toString idx}";
+    pubkey = ./. +  "/ssh_host_key.d/c1-${toString n}.pub";
+  })) nodes

ekman/c1/ssh_host_key.d/c1-1.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBEV8wEkeGUOs6umhdeOKYnVlYlta2rOCZSoezvu+bZ4 root@nixos

ekman/c1/ssh_host_key.d/c1-2.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDIsqJqWWftoiyiOOHnIuHYqbsaOg4AbKNm80wpjH2E8 root@nixos

ekman/c1/ssh_host_key.d/c1-3.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwwu4RJJhKo8s2Mtmpdvxs02d4IwrmS9cTpfAOQ2YkI root@nixos

ekman/c1/ssh_host_key.d/c1-4.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQFQqo2vcj2eYaH7nwdEzgCPme+7g3Db+s16KbQHzLI root@nixos

ekman/c1/ssh_host_key.d/c1-5.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImItUKMSygMY/1ZNsyGkyfywyngEDgZ7TxM63UwG1VH root@nixos

ekman/c1/ssh_host_key.d/c1-6.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBq+v2bBhQieqfXmtmGYb/9mOv9oc88zerRkkpGpoc0x root@nixos

ekman/c1/ssh_host_key.d/c1-7.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDme/xztp22wQOybtN2TVXMcn2QcVaXtRMp4AnPnzr2T root@nixos

ekman/c1/ssh_host_key.d/c1-8.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID1WxWnNg+L2+lQ3W2mD0/6kqwuCUOEJImvWnQYLzdUB root@nixos