wip: convert ekman to new cluster sturcture (not complete)
This commit is contained in:
Jonas Juselius
1
ekman/c0x/connauthfile
Normal file
1
ekman/c0x/connauthfile
Normal file
@@ -0,0 +1 @@
|
||||
莵q丘
|
||||
108
ekman/c0x/default.nix
Normal file
108
ekman/c0x/default.nix
Normal file
@@ -0,0 +1,108 @@
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
let
|
||||
# Pin the deployment package-set to a specific version of nixpkgs
|
||||
# pkgs = import (builtins.fetchTarball {
|
||||
# url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
|
||||
# sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
|
||||
# }) {};
|
||||
# pkgs = import <nixpkgs> {};
|
||||
|
||||
nodes = import ./nodes.nix;
|
||||
|
||||
compute = {
|
||||
deployment.tags = [ "compute" "c0" ];
|
||||
|
||||
fileSystems = {
|
||||
"/frontend" = {
|
||||
device = "10.255.241.100:/home";
|
||||
fsType = "nfs4";
|
||||
options = [
|
||||
"soft"
|
||||
"defaults"
|
||||
"noauto"
|
||||
"x-systemd.automount"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
systemd.automounts = [
|
||||
{
|
||||
where = "/frontend";
|
||||
wantedBy = [ "default.target" ];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
mkCompute = host:
|
||||
let
|
||||
hw = ./hardware-configuration.d + "/${host.name}.nix";
|
||||
in {
|
||||
"${host.name}" = {
|
||||
cluster = {
|
||||
compute = true;
|
||||
k8sNode = true;
|
||||
mounts = {
|
||||
rdma.enable = true;
|
||||
automount.enable = true;
|
||||
home = false;
|
||||
opt = true;
|
||||
work = true;
|
||||
data = true;
|
||||
backup = true;
|
||||
ceph = false;
|
||||
};
|
||||
};
|
||||
|
||||
features = {
|
||||
host = {
|
||||
name = host.name;
|
||||
address = host.address;
|
||||
};
|
||||
os.externalInterface = "enp33s0f0np0";
|
||||
hpc.compute = true;
|
||||
hpc.knem = true;
|
||||
# k8s = { inherit etcdCluster; };
|
||||
};
|
||||
|
||||
deployment.targetHost = host.address;
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
networking = {
|
||||
hostName = host.name;
|
||||
useDHCP = false;
|
||||
interfaces.enp33s0f3np3 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = host.address;
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
ipv4.routes = [ {
|
||||
address = "10.255.242.0";
|
||||
prefixLength = 24;
|
||||
via = "10.255.241.100";
|
||||
} ];
|
||||
|
||||
};
|
||||
# interfaces."ibp1s0.7666" = {
|
||||
interfaces."ibp1s0" = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = host.ipoib;
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
};
|
||||
imports = [
|
||||
hw
|
||||
../cluster.nix
|
||||
../mounts.nix
|
||||
# ./kernel.nix
|
||||
];
|
||||
}
|
||||
// compute;
|
||||
};
|
||||
in builtins.foldl' (a: n: a // mkCompute n) {} nodes
|
||||
|
||||
55
ekman/c0x/hardware-configuration.d/c0-17.nix
Normal file
55
ekman/c0x/hardware-configuration.d/c0-17.nix
Normal file
@@ -0,0 +1,55 @@
|
||||
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||
# and may be overwritten by future invocations. Please make changes
|
||||
# to /etc/nixos/configuration.nix instead.
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports =
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ "dm-snapshot" ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/46dfa481-ccd4-4e4f-b1d7-6875d582f7cd";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/F71C-0BD2";
|
||||
fsType = "vfat";
|
||||
};
|
||||
|
||||
# fileSystems."/data" =
|
||||
# { device = "/dev/disk/by-uuid/125c5d04-5266-43e8-887d-740d5944cb2b";
|
||||
# fsType = "xfs";
|
||||
# options = [
|
||||
# "noatime"
|
||||
# "nodiratime"
|
||||
# "logbufs=7"
|
||||
# "logbsize=255k"
|
||||
# "largeio"
|
||||
# "inode63"
|
||||
# "swalloc"
|
||||
# "allocsize=131071k"
|
||||
# ];
|
||||
# };
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
|
||||
# (the default) this is the recommended approach. When using systemd-networkd it's
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.docker0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
|
||||
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
44
ekman/c0x/hardware-configuration.d/c0-18.nix
Normal file
44
ekman/c0x/hardware-configuration.d/c0-18.nix
Normal file
@@ -0,0 +1,44 @@
|
||||
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||
# and may be overwritten by future invocations. Please make changes
|
||||
# to /etc/nixos/configuration.nix instead.
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports =
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ "dm-snapshot" ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/19d00648-5fbd-4464-93a0-e013d7f79d3a";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/68AF-2717";
|
||||
fsType = "vfat";
|
||||
};
|
||||
|
||||
fileSystems."/tmp" =
|
||||
{ device = "/dev/disk/by-uuid/bf1bede5-bc60-4603-874c-88ed85e6ab5f";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
|
||||
# (the default) this is the recommended approach. When using systemd-networkd it's
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
|
||||
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
48
ekman/c0x/kernel.nix
Normal file
48
ekman/c0x/kernel.nix
Normal file
@@ -0,0 +1,48 @@
|
||||
{pkgs, lib, ...}:
|
||||
let
|
||||
kernel = pkgs.linuxPackages.kernel;
|
||||
i40e =
|
||||
pkgs.stdenv.mkDerivation rec {
|
||||
name = "i40e-${version}-${kernel.version}";
|
||||
version = "2.13.10";
|
||||
|
||||
src = pkgs.fetchFromGitHub {
|
||||
owner = "dmarion";
|
||||
repo = "i40e";
|
||||
rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
|
||||
sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
|
||||
};
|
||||
|
||||
hardeningDisable = [ "pic" ];
|
||||
|
||||
configurePhase = ''
|
||||
cd src
|
||||
kernel_version=${kernel.modDirVersion}
|
||||
sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
|
||||
sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
|
||||
export makeFlags="BUILD_KERNEL=$kernel_version"
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
|
||||
'';
|
||||
|
||||
dontStrip = true;
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
meta = {
|
||||
description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
|
||||
homepage = https://github.com/dmarion/i40e;
|
||||
license = lib.licenses.gpl2;
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
# i40e2 = i40e;
|
||||
boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
|
||||
# overlay = self: super: {
|
||||
# linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
|
||||
# };
|
||||
}
|
||||
|
||||
11
ekman/c0x/nodes.nix
Normal file
11
ekman/c0x/nodes.nix
Normal file
@@ -0,0 +1,11 @@
|
||||
with builtins;
|
||||
let
|
||||
nodes = genList (n: n + 17) 2; in
|
||||
map (n: (
|
||||
rec {
|
||||
idx = 100 + n;
|
||||
name = "c0-${toString n}";
|
||||
address = "10.255.241.${toString idx}";
|
||||
ipoib = "10.255.243.${toString idx}";
|
||||
pubkey = ./. + "/ssh_host_key.d/c0-${toString n}.pub";
|
||||
})) nodes
|
||||
1
ekman/c0x/ssh_host_key.d/c0-17.pub
Normal file
1
ekman/c0x/ssh_host_key.d/c0-17.pub
Normal file
@@ -0,0 +1 @@
|
||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7wnYBwAdcp5+y7QOKzjd2apRv+Nvquy+qr3jVBeiQgbGsjlpGJBRsGv46QhSbB9QLuFuCUluuvilYbS9/ubm+BtRCrFViAHwFx+QhbLYlAXJPLGfXLxJZDSxbM6c9L+N+GAAfhiCfdhUVB39G33v5dd/VD3J4F/xC7lkPJU59pGokqe9dGrDta33pemSzR9QV9RuV4ZxJG1ld1B2L6TjSkAPTb79PPvMw8LnTViNffmHKp5qvwgIfwRvMApG5KnHw720pPncTW4ejY+utSbP4DcwD2qayKbxnsYDpimMDQxR3id6meZa5LcilxExmz/ig7Fpc3TB3wIpT4ha6VVu2FhRThKR5zD1Qif8BaKy5eAmqcWKQm6M9W56LmUdn/OHpzmaSv47UgtJ+1JG4LqGSaShs8kfVWtaqmev6vvnHBZMm7ni3xyROTSWnjJHLDQJbCYb1PBbct2Dod6MuU54CJUuxw0k+hCJ1QMz1ibme3+LauXVuLJ2JrnAoz7x2CcUv9e5MVx0H4HpWTCxi7cNBO5bbeuxMwHbl3JrZ58LBK9zZYl+nr2F7CZqLyoMimQuVpXT6Aw0k/nfYMAXjUJ3qbBldiGsRgpgbvL27nJK0EhQ8Kd36QhHfdJKhiAIPV+06WVWhPllaqay4OgZIQM1aB83eHg/OUxbOWEu89d7w/w== root@nixos
|
||||
1
ekman/c0x/ssh_host_key.d/c0-18.pub
Normal file
1
ekman/c0x/ssh_host_key.d/c0-18.pub
Normal file
@@ -0,0 +1 @@
|
||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/a5Q1Y8hta/v8W/cM45o6UimPLfSzz6NM+WaNAYbwnLoz2VVowHdjyndajmH1CMK/cTDRiVSrTkky48f3JNKzCMjmwVA6PjWmRxF0tqcitW9hzFB+L10+fgBsZu7MUMhuWfTtekIgnRjqNZIA3q5PGz9JrMHsRem+RGmDayaNSAkOZ1ToGSS7pnGm06qAswLJaK15wnupw5x9NYO1RXq2Dy2pX68e4urA+h9X9tqDfb6kQP3/9gb+q6BpiGUwUHW5x0HK/LwNqThKV7ZwzgMHqz4rq5TieGemTw8LKcDjj3BWQTDt4SlZ2gux49C7z1vBipHtxKHIdZH9AXYPZCkjgI/OVVJvxkbgMM+3FSr3j/KvTOdL5ycJvGBsF4dJnRTTpay7OmnksQ62+vwNO2dvDnhAgVntrr9Fb1ahLnTN7dOf6BlkUbz+vtYby0gcjr5sW8b4duGpJThDDkAGLUKkx1z94/bpPQpUb1ajXPizO7jjJCK20zTXWG2uiOdXWHmZ3YVoJ/qSOvtUAX2RR2FgJzHqnDnHtmLiMc9OkZ0eJwo6ltBqGHPNXCyBwHAa8eSH2Af+10vfzJ88J8MGD88DV7SYWbJxuB+KMAnDX04UfrvqLnRADzbFuiG8locorXFE90wEZTz/nFWsM+Q/XPA+M1GaO6fK9SyN45LgApKARQ== root@nfs1
|
||||
Reference in New Issue
Block a user