oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

major: initial rossy cluster and biggish refactor

Browse Source
This commit is contained in:
Jonas Juselius
2025-09-12 11:59:15 +02:00
parent 899a7f4338
commit e0846164a7
120 changed files with 875 additions and 1746 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
rossby/bin/adduser.sh Executable file
View File

@@ -0,0 +1,29 @@
#!/usr/bin/env bash
id=$1
user=$2
name="$3"
grp="\ $user = { gid = "$id"; };"
read -d '' usr << EOF
\\\ $user = {\\\n\
description = "$name";\\\n\
home = "/home/$user";\\\n\
group = "$user";\\\n\
extraGroups = [\\\n\
"users"\\\n\
"docker"\\\n\
];\\\n\
uid = $id;\\\n\
isNormalUser = true;\\\n\
createHome = true;\\\n\
openssh.authorizedKeys.keys = [];\\\n\
};\\\n\
EOF
sed -i "
/# @grp@/i $grp
/# @usr@/i $usr
" stokes/users.nix

14
rossby/bin/copy-hardware-configuration.sh Executable file
View File

@@ -0,0 +1,14 @@
#!/usr/bin/env bash
TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
if [ $# != 1 ]; then
echo "usage: copy-hardware-configuration.sh name"
exit 1
fi
node=$1
[ -e $node.nix ] && mv $node.nix $node.nix.bak
scp root@$node:/etc/nixos/hardware-configuration.nix $node.nix

27
rossby/bin/deploy.sh Executable file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env bash
TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
if [ $# = 0 ]; then
echo "usage: deploy.sh name ..."
exit 1
fi
if [ ! -f $TOP/$1/default.nix ]; then
echo "error: $1 does not contain a deployment"
exit 1
fi
cd $TOP/$1
nixops list | grep -q $1
if [ $? = 0 ]; then
echo "--- Updating deployment"
nixops modify -d $1 .
else
echo "--- Creating deployment"
nixops create -d $1 .
fi
echo "--- Deploying $1"
nixops deploy -k -d $* --allow-reboot

18
rossby/bin/initca.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
if [ "x$1" = "x" ]; then
echo "usage: initca.sh {cluster}"
exit 1
fi
ca=$TOP/modules/initca.nix
cd $TOP/$1
echo "--- Preparing CA certificate"
nix-build -o ca $ca
echo "--- Safeguarding CA certificate"
nix-store --add-root $(pwd)/ca --indirect -r $(nix-instantiate --add-root $ca)

12
rossby/bin/reboot.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
if [ $# = 0 ]; then
echo "usage: reboot.sh cluster "
exit 1
fi
d=$1
shift
nixops reboot -d $d $*

12
rossby/bin/ssh.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
if [ $# = 0 ]; then
echo "usage: ssh.sh cluster ..."
exit 1
fi
d=$1; shift
nixops ssh-for-each -d $d -- $@

45
rossby/bin/teardown.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
reboot=no
case $1 in
--reboot) reboot=yes; shift ;;
esac
if [ $# != 1 ]; then
echo "usage: teardown.sh [--reboot] name"
exit 1
fi
d=$1
tmp=$TOP/.$d.$$
teardown () {
mkdir -p $tmp
cp -r $TOP/$d/* $tmp
sed -i '/k8s *= *{/,+1 s/enable *= *true/enable = false/' $tmp/cluster.nix
nixops modify -d $d $tmp
nixops deploy -d $d
[ $reboot = yes ] && nixops reboot -d $d
nixops ssh-for-each -d $d \
"rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl"
rm -rf $tmp
}
cat << EOF
************************************************************************
*** ***
*** WARNING: This will irrevokably destroy the running cluster! ***
*** ***
************************************************************************
EOF
echo "Are you sure you want to tear down $d? (YES/no)"
read a
case $a in
YES) teardown ;;
*) echo "Bailing out." ;;
esac

1
rossby/c0/connauthfile Normal file
View File

@@ -0,0 +1 @@
莵q丘

106
rossby/c0/default.nix Normal file
View File

@@ -0,0 +1,106 @@
{ pkgs ? import <nixpkgs> {} }:
let
# Pin the deployment package-set to a specific version of nixpkgs
# pkgs = import (builtins.fetchTarball {
# url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
# sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
# }) {};
# pkgs = import <nixpkgs> {};
nodes = import ./nodes.nix;
compute = {
deployment.tags = [ "compute" "c0" ];
fileSystems = {
"/users" = {
device = "172.16.239.222:/home";
fsType = "nfs4";
options = [
"soft"
"defaults"
"noauto"
"x-systemd.automount"
];
};
};
systemd.automounts = [
{
where = "/users";
wantedBy = [ "default.target" ];
}
];
};
mkCompute = host:
let
hw = ./hardware-config.d + "/${host.name}.nix";
in {
"${host.name}" = {
cluster = {
compute = true;
k8sNode = true;
mounts = {
rdma.enable = false;
automount.enable = true;
users = true;
opt = true;
work = false;
data = false;
ceph = false;
};
};
features = {
host = {
name = host.name;
address = host.address;
};
os.externalInterface = "enp65s0f0";
hpc.compute = true;
# hpc.knem = true;
};
deployment.targetHost = host.address;
# services.udev.extraRules = ''
# KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
networking = {
hostName = host.name;
useDHCP = false;
interfaces. enp65s0f0 = {
useDHCP = false;
ipv4.addresses = [ {
address = host.address;
prefixLength = 24;
} ];
# ipv4.routes = [ {
# address = "10.255.242.0";
# prefixLength = 24;
# via = "10.255.241.100";
# } ];
};
# interfaces."ibp1s0.7666" = {
# interfaces."ibp1s0" = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = host.ipoib;
# prefixLength = 24;
# } ];
# };
};
imports = [
hw
../default.nix
../mounts.nix
#./kernel.nix
];
}
// compute;
};
in builtins.foldl' (a: n: a // mkCompute n) {} nodes

43
rossby/c0/hardware-config.d/c0-10.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/9e236fcb-00e5-43a7-ad4b-41b1b068cd5d";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/28C7-DCBE";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-11.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/92a043ce-b2be-4cdd-b6e0-1e8f3e6952fd";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/3038-56D0";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-13.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/f0909558-f71a-472e-af38-8ffdb2e9e453";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/72A0-CF21";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-14.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/c35b6b6f-29da-4ea0-8fba-067301ce9572";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/3FC3-84E2";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-15.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/1a765ce5-2156-4427-8594-824473a31205";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/4CD7-6D3E";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-16.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/2580dfeb-a28e-4f3d-87a8-e7c7a9fdfeca";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/6BB5-3DC2";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-17.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/5d8be7db-bdb6-47d2-bb9a-5dc62f162c0f";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/8C14-D452";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-18.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/0a0feff4-c3aa-4a67-9880-78c08f2e1db3";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/A52D-B7FA";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-19.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/cd6d1033-a92f-4ab7-8265-00d57466221b";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/AEA2-6A41";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-2.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/032fe2e6-9091-47e8-b121-0123abe4f0fc";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/EE11-AD60";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-20.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/7dab9648-6618-46df-978e-f0ac43405fd5";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/08D3-CB99";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-3.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/da170cd8-0a6b-470f-96aa-a7bdf3dd8736";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/159D-E4C1";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-4.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/b081d8a1-3cae-4017-afd3-0482d58f3062";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/1E63-EA5E";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-5.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/494bceb1-c0d1-419d-a379-8104ff90aa61";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/294F-20BD";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-6.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/be0a7204-1a90-4f06-980a-a315e9c664ca";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/4889-6E82";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

43
rossby/c0/hardware-config.d/c0-8.nix Normal file
View File

@@ -0,0 +1,43 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/ffeb8ede-16f1-48bc-a976-1ea9ab982745";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/E780-1C40";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

44
rossby/c0/hardware-config.d/c0-9.nix Normal file
View File

@@ -0,0 +1,44 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/3d27bbdb-4370-4af2-9747-74d03acfa558";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/9E1B-2495";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true;
# networking.interfaces.tailscale0.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

58
rossby/c0/kernel.nix Normal file
View File

@@ -0,0 +1,58 @@
{pkgs, lib, ...}:
let
kernel = pkgs.linuxPackages.kernel;
i40e =
pkgs.stdenv.mkDerivation rec {
name = "i40e-${version}-${kernel.version}";
version = "2.13.10";
src = pkgs.fetchFromGitHub {
owner = "dmarion";
repo = "i40e";
rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
};
hardeningDisable = [ "pic" ];
configurePhase = ''
cd src
kernel_version=${kernel.modDirVersion}
sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
export makeFlags="BUILD_KERNEL=$kernel_version"
'';
installPhase = ''
install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
'';
dontStrip = true;
enableParallelBuilding = true;
meta = {
description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
homepage = https://github.com/dmarion/i40e;
license = lib.licenses.gpl2;
};
};
in
{
# i40e2 = i40e;
# boot.kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override {
# argsOverride = rec {
# src = pkgs.fetchurl {
# url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
# sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid";
# };
# version = "5.10.239";
# modDirVersion = "5.10.239";
# };
# });
boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
# overlay = self: super: {
# linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
# };
}

11
rossby/c0/nodes.nix Normal file
View File

@@ -0,0 +1,11 @@
with builtins;
let
nodes = genList (n: n + 1) 20; in
map (n: (
rec {
idx = 110 + n;
name = "c0-${toString n}";
address = "172.16.239.${toString idx}";
ipoib = "172.16.240.${toString idx}";
pubkey = ./. + "/ssh_host_key.d/c0-${toString n}.pub";
})) (filter (x: if x == 1 || x == 7 || x == 12 then false else true) nodes)

1
rossby/c0/ssh_host_key.d/c0-10.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBd59La09RhuP0kiYXSGqAIY9ZV9lKzMOuqXUCHfNuRA root@nixos

1
rossby/c0/ssh_host_key.d/c0-11.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcaGpXQqp/3WwI0iJlHuIuNvq60Pan8mj2Jri2J8V0W root@nixos

1
rossby/c0/ssh_host_key.d/c0-13.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICAEVDoSKJc41wkIOfsYcKezmQwLADtcU5WUGz7+gWP9 root@nixos

1
rossby/c0/ssh_host_key.d/c0-14.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdrfN8vG/UUjUj+JrXAnfS7wYmiO+mmuTEElfUDOSTA root@nixos

1
rossby/c0/ssh_host_key.d/c0-15.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHil3FeI6GxrpJ52XlZDr6FBQ0gdjX4WrnLH5bHmSYXz root@nixos

1
rossby/c0/ssh_host_key.d/c0-16.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGaHMHXeHdL/rOaVWPVPx7rpL6UWBiOaHKOaQdzOFc/3 root@nixos

1
rossby/c0/ssh_host_key.d/c0-17.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF+YvVsZwsFeyghA4hG6o0ja+zjN3d42p//8fwOuYyDQ root@nixos

1
rossby/c0/ssh_host_key.d/c0-18.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMkeYwO3dGG+l3tmTvP23xroT2Dm+aZb46ISIjzWvIkk root@nixos

1
rossby/c0/ssh_host_key.d/c0-19.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKTKLGyYiuRMUNqim4geRv6bramCigPdzA3Ed4Meupkd root@nixos

1
rossby/c0/ssh_host_key.d/c0-2.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBnQW4aNsmDrNu8yXa8xJtKNWi9Cf2BHZ7Bc/SP26zs root@nixos

1
rossby/c0/ssh_host_key.d/c0-20.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILELid8+ax9zHcm3Jbv5oE1XMVmqyjmR+/KyA+thd5v4 root@nixos

1
rossby/c0/ssh_host_key.d/c0-3.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHydtfeTvjqWDQnMPdLAWwirZvV6DrBe2K7yZ3lR1oQy root@nixos

1
rossby/c0/ssh_host_key.d/c0-4.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO+0beDz0Tk3wuBTk8miH/NKE4U/IbHPq/qMj+y644iV root@nixos

1
rossby/c0/ssh_host_key.d/c0-5.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC+23znDppuQCDRgpoovlGUG113nZ9vFKrBST8sf0BwP root@nixos

1
rossby/c0/ssh_host_key.d/c0-6.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJzUH3PJi8v0vQfzj2WXsi9IM/0MVpZ87Clu8LyCepZj root@nixos

1
rossby/c0/ssh_host_key.d/c0-8.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICecn4QgdQmK/tHB1Q5EhitExEuVm+JtnyB7eOqIwGrj root@nixos

1
rossby/c0/ssh_host_key.d/c0-9.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICf1lE0k49jPMxixAfAwqC2s3Wov8k3P8WgSQ6QTUHO9 root@rossby-manage

321
rossby/default.nix Normal file
View File

@@ -0,0 +1,321 @@
{ pkgs, lib, config, ... }:
with lib;
let
cfg = config.features.host;
computeNodes = import ./c0/nodes.nix;
mkSANs = host: [
host.name
host.address
"127.0.0.1"
];
configuration = {
system.autoUpgrade.enable = lib.mkForce false;
nixpkgs.overlays = [
(import ./overlays.nix)
(import ../modules/overrides/certmgr.nix)
];
boot = {
loader.systemd-boot.enable = true;
loader.efi.canTouchEfiVariables = true;
# kernelPackages = pkgs.linuxKernel.packages.linux_6_9;
kernelModules = [ "ib_umad" "ib_ipoib" "ceph" ];
# kernelParams = [
# "console=ttyS0,115200"
# "console=tty0"
# ];
};
console = {
font = "Lat2-Terminus16";
keyMap = "us";
};
i18n = {
defaultLocale = "en_US.UTF-8";
extraLocaleSettings = {
LC_CTYPE="en_DK.UTF-8";
LC_TIME="en_DK.UTF-8";
LC_PAPER="en_DK.UTF-8";
LC_NAME="en_DK.UTF-8";
LC_ADDRESS="en_DK.UTF-8";
LC_TELEPHONE="en_DK.UTF-8";
LC_MEASUREMENT="en_DK.UTF-8";
LC_IDENTIFICATION="en_DK.UTF-8";
};
};
time.timeZone = "Europe/Oslo";
# programs.msmtp = {
# enable = false;
# accounts = {
# default = {
# auth = false;
# tls = false;
# tls_starttls = false;
# port = 24;
# from = "rossby@oceanbox.io";
# host = "smtpgw.itpartner.no";
# # user = "utvikling";
# # password = "S0m3rp0m@de#21!";
# };
# };
# defaults = {
# aliases = "/etc/aliases";
# };
# };
environment.etc = {
"aliases" = {
text = ''
root: jonas.juselius@oceanbox.io
'';
mode = "0644";
};
};
features = {
os = {
# boot.uefi = true;
adminAuthorizedKeys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas"
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHVwcJOtx9YTWy+aD4xGbyPFLOdMN6NqY8wcfDtHczyT Stig Rune Jensen"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfgY468dPNpdXZCkD9jw1p2qA0+z56Wi/c1VYE+riki Stig Rune Jensen"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst"
];
docker.enable = false;
};
cachix.enable = false;
monitoring.nodeExporter.enable = false;
hpc.mft.enable = false; # Mellanox MFT
};
networking = {
domain = mkDefault "cluster.local";
defaultGateway = mkDefault "172.16.239.1";
nameservers = mkDefault [ "8.8.8.8" ];
search = mkDefault [];
extraHosts = import ./hosts.nix;
firewall.extraCommands = ''
iptables -I INPUT -s 172.16.239.0/24 -j ACCEPT
iptables -I INPUT -s 172.16.240.0/24 -j ACCEPT
'';
};
environment.variables = {};
# systemd.services."serial-getty@ttyS0".enable = true;
# environment.etc."beegfs/connauthfile" = {
# source = ./connauthfile;
# mode = "0400";
# uid = 0;
# gid = 0;
# };
services.certmgr.validMin = "120h";
services.certmgr.renewInterval = "30m";
nix.settings = {
max-jobs = 32;
trusted-users = [ "@wheel" ];
substituters = [
];
};
system.activationScripts = {
kraken-permissions.text = ''
chmod 755 /work/kraken
'';
};
};
slurm = {
features.hpc.slurm = {
enable = true;
client = true;
mungeKey = ./munge.key;
mungeUid = mkDefault 994; # hack
# pkey = "0x7666";
controlMachine = "rossby-manage";
mailDomain = "oceanbox.io";
nodeName = [
"c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
"rossby-login Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
"rossby-manage Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
];
partitionName = [
"batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP"
"rossby Nodes=rossby-login MaxTime=1:00:00 State=UP"
"short Nodes=c0-[1-8] MaxTime=INFINITE State=UP"
"long Nodes=c0-[3-8] MaxTime=INFINITE State=UP"
"stats Nodes=c0-[7-8] MaxTime=INFINITE State=UP"
];
};
};
compute = {
system.activationScripts = {
mkWorkDir.text = "mkdir -p /work";
};
cluster.slurm = true;
features = {
hpc = {
enable = true;
beegfs = {
enable = false;
# beegfs = {
# work = {
# mgmtdHost = "ibbeegfs0";
# connAuthFile = "/etc/beegfs/connauthfile";
# client = {
# enable = false;
# mountPoint = "/work";
# };
# };
# };
};
};
};
};
k8s-node = {
features = {
k8s = {
enable = true;
node.enable = true;
clusterName = "rossby";
initca = ./ca;
cidr = "10.10.0.0/16";
master = {
name = "rossby-manage";
address = "172.16.239.221";
# extraSANs = [
# "rossby-manage.oceanbox.io"
# ];
};
ingressNodes = [
"rossby-manage.oceanbox.io"
"rossby.oceanbox.io"
];
fileserver = "rossby-login";
charts = {
acme_email = "acme@oceanbox.io";
# grafana_smtp_user = "utvikling";
# grafana_smtp_password = "S0m3rp0m@de#21!";
};
};
};
system.activationScripts = {
copyCaKey.text = "cp ${./ca}/ca-key.pem /var/lib/kubernetes/secrets";
};
# services.kubernetes.kubelet.extraSANs = mkSANs {
# name = cfg.name;
# address = cfg.address;
# };
};
shosts = {
environment.etc."ssh/shosts.equiv" = {
mode = "0644";
uid = 0;
gid = 0;
text = ''
172.16.239.210
'' + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes;
};
programs.ssh.knownHosts = {
rossby-manage = {
hostNames = [
"manage" "manage.compute.local" "rossby-manage.oceanbox.io" "172.16.239.221" "172.16.240.221"
];
publicKeyFile = ./manage/ssh_host_key.pub;
};
rossby-login = {
hostNames = [
"rossby" "rpssby-login" "rossby-login.compute.local" "rossby.compute.local" "rossby.oceanbox.io" "172.16.239.222" "172.16.240.222"
];
publicKeyFile = ./login/ssh_host_key.pub;
};
# fs-work = {
# hostNames = [
# "fs-work" "fs-work.compute.local" "172.16.239.90" "172.16.240.210"
# ];
# publicKeyFile = ./fs-work/ssh_host_key.pub;
# };
} // builtins.foldl' (a: x:
let n = toString x.idx;
in a // {
"${x.name}" = {
hostNames = [
"${x.name}"
"${x.name}.compute.local"
"172.16.239.${n}"
"172.16.240.${n}"
];
publicKeyFile = x.pubkey;
};
}) {} computeNodes;
environment.systemPackages = [
openssh-shosts
pkgs.inotify-tools
pkgs.ceph
pkgs.ceph-client
];
security.wrappers = {
ssh-keysign = {
source = "${openssh-shosts}/libexec/ssh-keysign";
owner = "root";
group = "root";
permissions = "u+rs,g+rx,o+rx";
};
};
};
openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {
buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ];
doCheck = false; # the tests take hours
});
in {
options.cluster = {
compute = mkEnableOption "Enable compute node configs";
};
options.cluster = {
k8sNode = mkEnableOption "Enable k8s node";
};
options.cluster = {
slurm = mkEnableOption "Enable slurm";
};
config = mkMerge [
configuration
shosts
(mkIf config.cluster.compute compute)
(mkIf config.cluster.k8sNode k8s-node)
(mkIf config.cluster.slurm slurm)
];
imports = [
../modules
../nixos
../users.nix
];
}

12
rossby/etcdCluster.nix Normal file
View File

@@ -0,0 +1,12 @@
{
enable = true;
existing = false;
nodes =
{
# manage = "172.16.239.221";
# rossby = "172.16.239.222";
# fs-work = "172.16.239.210";
};
}

171
rossby/fs-work/default.nix Normal file
View File

@@ -0,0 +1,171 @@
{ pkgs ? import <nixpkgs> {} }:
let
# Pin the deployment package-set to a specific version of nixpkgs
# pkgs = import (builtins.fetchTarball {
# url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
# sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
# }) {};
# pkgs = import <nixpkgs> {};
etcdCluster = import ../etcdCluster.nix;
name = "fs-work";
address = "10.255.241.90";
in {
fs-work = { config, pkgs, ... }: with pkgs; {
deployment.tags = [ "fs" "fs-work" ];
deployment.targetHost = address;
system.autoUpgrade.enable = lib.mkForce false;
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
};
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
environment.systemPackages = with pkgs; [
rdma-core
hwloc
];
cluster = {
k8sNode = true;
slurm = false;
mounts = {
rdma.enable = false;
automount.enable = true;
users = false;
opt = false;
work = false;
data = false;
ceph = false;
};
};
features.hpc.slurm.mungeUid = 994;
features = {
host = {
inherit address;
inherit name;
};
os = {
externalInterface = "enp33s0f3np3";
nfs.enable = true;
nfs.exports = ''
/exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
k8s = {
enable = true;
node.enable = true;
master.enable = false;
inherit etcdCluster;
};
};
systemd.services.rc-local = {
description = "rc.local script";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
path = [ "/run/current-system/sw/" ];
serviceConfig = {
Type = "oneshot";
};
script = ''
# if [ -e /sys/block/md126 ]; then
# echo "deadline" > /sys/block/md126/queue/scheduler
# # echo "4096" > /sys/block/md126/queue/nr_requests
# echo "4096" > /sys/block/md126/queue/read_ahead_kb
# echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
# echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
# fi
grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
'';
};
boot.kernel.sysctl = {
"vm.dirty_background_ratio" = 5;
"vm.dirty_ratio" = 10;
"vm.vfs_cache_pressure" = 50;
"vm.min_free_kbytes" = 262144;
};
networking = {
hostName = name;
interfaces.enp65s0f0np0 = {
useDHCP = false;
ipv4.addresses = [
{
address = address;
prefixLength = 24;
}
];
ipv4.routes = [
{
address = "10.255.242.0";
prefixLength = 24;
via = "10.255.241.100";
}
];
};
interfaces.enp1s0f1np1 = {
useDHCP = false;
ipv4.addresses = [
{
address = "10.255.244.90";
prefixLength = 24;
}
];
};
# interfaces."ibp65s0.7666" = {
# useDHCP = false;
# };
interfaces.ibp1s0f0 = {
useDHCP = false;
ipv4.addresses = [
{
address = "10.255.243.90";
prefixLength = 24;
}
];
};
firewall = {
allowedTCPPorts = [];
allowedUDPPorts = [];
extraCommands = ''
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';
};
};
services.rpcbind.enable = true;
fileSystems = {
"/exports/work" = {
device = "/work";
options = [ "bind" ];
};
"/exports/opt" = {
device = "/opt";
options = [ "bind" ];
};
};
programs.singularity.enable = true;
imports = [
./hardware-configuration.nix
../default.nix
../mounts.nix
];
};
}

47
rossby/fs-work/hardware-configuration.nix Normal file
View File

@@ -0,0 +1,47 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "smartpqi" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
boot.initrd.kernelModules = [ "dm-snapshot" ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/21d2c2c7-4968-432d-a5c4-08116147b816";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/49B0-2597";
fsType = "vfat";
options = [ "fmask=0022" "dmask=0022" ];
};
fileSystems."/work" =
{ device = "/dev/mapper/data-work";
fsType = "xfs";
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0f0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0f1np1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0f1np1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp6s0f4u1u6.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

1
rossby/fs-work/ssh_host_key.pub Normal file
View File

@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxcJ+G4e6TEy0CZ66V6rRvODZ3QBgG5uIPElZr0lAFEe2FLPtiR57puXTV4tFhkY/TsVo0BE5/qDcBUzdRJ0ihuVVruvEUJDr+wDby1+V823xvPYT0IqPNvBBcR/FKIUTWFIhDHCntQ3MWvcp+bXX4ZH02QGmnSNNjyH6krOofuNz0/H8UxpLn/U40ruQx5SyT+uc7t0qryEUAwdvkQ6dGqyeCd4szSyQ1RfaTcd3qqXltxfKx5EbRAxD6pLoiKzYBGzWwKOLczNJGsLoMQ53KhEUyAdEOEx79qxoR4PU8P5q7osZD9NuloHgVT2P1hiNcsNmFzze3WW96PyK9L6Y1NE2rfcrhHIHG8tK31YXBlFQWr/VuPHpvqt21z0r2tPJwM5ui1zgZ1tsEWuKURItqr5Xsxk3D3woY5hT4rtDkV0TLe8/PUTGkGtw15UbQ/BnsFctWe5JnTtZ+90Tjxl4Ty/h0AYilVff6I3tfEln9MBnYig/cOUX0lmo2lnYpUedHYq5LXaga15OW0uonMd8Wl0nSGbqpJ0Gmu+gONJU9AbGchgxxmo0LNbdQ4jANq11LhcK2UZpyPKA7zV0pcgo1WSQyewfdFYIOJzGJPbDhOai/0OAXJYPJa/QJvp08ULpGrUUYXtjmVLw0DCYihm4af84Gr5Fu7hqNO1dplbJh8w== root@nixos

32
rossby/hosts.nix Normal file
View File

@@ -0,0 +1,32 @@
''
# 172.16.241.2 rossby-gw rossby-gw.cluster.local rossby-gw.cluster.local
# 172.16.241.3 front-gw front-gw.cluster.local front-gw.cluster.local
172.16.239.221 rossby-manage rossby-manage.cluster.local manage.cluster.local
172.16.239.222 rossby rossby-login rossby-login.cluster.local rossby.cluster.local
172.16.239.210 fs-work fs-work.cluster.local
172.16.239.221 etcd0 etcd0.cluster.local
172.16.239.222 etcd1 etcd1.cluster.local
172.16.239.210 etcd2 etcd2.cluster.local
172.16.239.111 c0-1 c0-1.cluster.local
172.16.239.112 c0-2 c0-2.cluster.local
172.16.239.113 c0-3 c0-3.cluster.local
172.16.239.114 c0-4 c0-4.cluster.local
172.16.239.115 c0-5 c0-5.cluster.local
172.16.239.116 c0-6 c0-6.cluster.local
172.16.239.117 c0-7 c0-7.cluster.local
172.16.239.118 c0-8 c0-8.cluster.local
172.16.239.119 c0-9 c0-9.cluster.local
172.16.239.120 c0-10 c0-10.cluster.local
172.16.239.121 c0-11 c0-11.cluster.local
172.16.239.122 c0-12 c0-12.cluster.local
172.16.239.123 c0-13 c0-13.cluster.local
172.16.239.124 c0-14 c0-14.cluster.local
172.16.239.125 c0-15 c0-15.cluster.local
172.16.239.126 c0-16 c0-16.cluster.local
172.16.239.127 c0-17 c0-17.cluster.local
172.16.239.128 c0-18 c0-18.cluster.local
172.16.239.129 c0-19 c0-18.cluster.local
172.16.239.130 c0-20 c0-18.cluster.local
''

334
rossby/login/default.nix Normal file
View File

@@ -0,0 +1,334 @@
{ pkgs ? import <nixpkgs> {} }:
let
name = "rossby";
address = "172.16.239.222";
etcdCluster = import ../etcdCluster.nix;
in
{
rossby-login = { config, pkgs, ... }: with pkgs; {
deployment.tags = [ "rossby-login" "login" ];
deployment.targetHost = address;
system.autoUpgrade.enable = lib.mkForce false;
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
};
cluster = {
compute = true;
k8sNode = true;
mounts = {
rdma.enable = false;
automount.enable = true;
users = false;
opt = false;
work = false;
data = false;
ceph = false;
};
};
features = {
host = {
inherit name;
inherit address;
};
myvnc.enable = false;
os = {
externalInterface = "enp129s0f0";
nfs.enable = true;
nfs.exports = ''
/exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
hpc = {
slurm.server = false;
slurm.slurmrestd = false;
manager = false;
login = true;
knem = false;
};
k8s = {
master.enable = false;
node.enable = true;
inherit etcdCluster;
};
desktop.enable = false;
# server.enable = true;
monitoring = {
# server = {
# enable = false;
# scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
# defaultAlertReceiver = {
# email_configs = [
# { to = "jonas.juselius@oceanbox.io"; }
# ];
# };
# pageAlertReceiver = {
# webhook_configs = [
# {
# url = "https://prometheus-msteams.k2.itpartner.no/rossby";
# http_config = {
# tls_config = { insecure_skip_verify = true; };
# };
# }
# ];
# };
# };
# webUI.enable = false;
# webUI.acmeEmail = "innovasjon@itpartner.no";
# webUI.allow = [
# "10.1.2.0/24"
# "172.19.254.0/24"
# "172.19.255.0/24"
# ];
infiniband-exporter = {
enable = true;
nameMap = ''
# 0xe8ebd3030024981e "c0-1"
'';
};
slurm-exporter = {
enable = true;
port = 6080;
};
};
};
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
services.flannel.iface = "enp129s0f0";
networking = {
useDHCP = false;
hostName = name;
interfaces.enp129s0f0 = {
useDHCP = false;
ipv4.addresses = [ {
inherit address;
prefixLength = 24;
} ];
# ipv4.routes = [
# {
# address = "10.255.244.0";
# prefixLength = 24;
# via = "10.255.241.99";
# }
# ];
};
# interfaces."ibp65s0f0" = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = "10.255.243.100";
# prefixLength = 24;
# } ];
# };
# interfaces."enp65s0f1np1" = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = "10.255.244.100";
# prefixLength = 24;
# } ];
# };
# interfaces.enp33s0f0np0 = {
# useDHCP = false;
# ipv4.addresses = [ {
# address = "10.255.242.2";
# prefixLength = 24;
# } ];
# ipv4.routes = [
# {
# address = "10.1.8.0";
# prefixLength = 24;
# via = "10.255.242.1";
# }
# {
# address = "10.1.30.0";
# prefixLength = 24;
# via = "10.255.242.1";
# }
# ];
# };
defaultGateway = "172.16.239.1";
firewall = {
allowedTCPPorts = [ 4443 ];
extraCommands = ''
# iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';
};
};
fileSystems = {
"/exports/home" = {
device = "/home";
options = [ "bind" ];
};
"/exports/opt/bin" = {
device = "/opt/bin";
options = [ "bind" ];
};
"/exports/opt/sif" = {
device = "/opt/sif";
options = [ "bind" ];
};
"/exports/opt/singularity" = {
device = "/opt/singularity";
options = [ "bind" ];
};
"/exports/nfs-provisioner" = {
device = "/vol/nfs-provisioner";
options = [ "bind" ];
};
"/users" = {
device = "/home";
options = [ "bind" ];
};
"/vol/local-storage/vol1" = {
device = "/vol/vol1";
options = [ "bind" ];
};
"/vol/local-storage/vol2" = {
device = "/vol/vol2";
options = [ "bind" ];
};
};
nix.extraOptions = ''
# secret-key-files = /etc/nix/rossby.key
'';
# services.xserver = {
# enable = false;
# enableCtrlAltBackspace = true;
# layout = "us";
# xkbVariant = "altgr-intl";
# xkbOptions = "eurosign:e";
# displayManager = {
# gdm.enable = false;
# job.logToFile = true;
# };
# # desktopManager.xfce.enable = true;
# };
services.prometheus.alertmanager.configuration.global = {
smtp_smarthost = "smtpgw.itpartner.no";
# smtp_auth_username = "utvikling";
# smtp_auth_password = "S0m3rp0m@de#21!";
smtp_hello = "rossby.oceanbox.io";
smtp_from = "noreplyrossby.oceanbox.io";
};
# services.nginx = {
# virtualHosts = {
# "ds.matnoc.regnekraft.io" = {
# forceSSL = true;
# enableACME = true;
# serverAliases = [];
# locations."/" = {
# proxyPass = "http://localhost:9088";
# proxyWebsockets = false;
# extraConfig = ''
# allow 10.1.2.0/24;
# allow 172.19.254.0/24;
# allow 172.19.255.0/24;
# deny all;
# '';
# };
# };
# };
# };
# services.gitlab-runner = {
# enable = true;
# extraPackages = with pkgs; [
# singularity
# ];
# concurrent = 4;
# services = {
# sif = {
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
# executor = "shell";
# tagList = [ "rossby" "sif" ];
# };
# };
# };
# security.sudo.extraConfig = ''
# gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
# '';
security.pam = {
services.sshd.googleAuthenticator.enable = true;
loginLimits = [
{
domain = "@users";
item = "rss";
type = "hard";
value = 16000000;
}
{
domain = "@users";
item = "cpu";
type = "hard";
value = 180;
}
];
};
system.activationScripts = {
home-permissions.text = ''
chmod 755 /home/olean
chmod 755 /home/frankgaa
chmod 755 /home/jonas
chmod 755 /home/mrtz
chmod 755 /home/avle
chmod 755 /home/stig
chmod 755 /home/bast
chmod 755 /home/simenlk
chmod 755 /work/kraken
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it
services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
PubkeyAuthOptions verify-required
'';
environment.systemPackages = [];
virtualisation.docker.enable = pkgs.lib.mkForce true;
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "server"; # for exit-node usage
extraUpFlags = [
"--login-server=https://headscale.svc.oceanbox.io"
"--accept-dns"
"--advertise-exit-node"
"--advertise-tags=tag:rossby"
];
};
imports = [
./hardware-configuration.nix
../default.nix
../mounts.nix
../myvnc.nix
];
};
}

42
rossby/login/hardware-configuration.nix Normal file
View File

@@ -0,0 +1,42 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/e859992d-1627-42cc-a59c-178d596fc220";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/5281-9FFF";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f3.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0np0.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

1
rossby/login/ssh_host_key.pub Normal file
View File

@@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE83l0a2zm4kVF3Dr42SUeSutc1F8dpNvkEtUg3wUWUf root@nixos

331
rossby/manage/default.nix Normal file
View File

@@ -0,0 +1,331 @@
{ pkgs, ...}:
let
computeNodes =
import ../c0/nodes.nix ++
[
rec {
idx = 222;
name = "rossby-login";
address = "172.16.239.${toString idx}";
ipoib = "172.16.240.${toString idx}";
pubkey = ../login/ssh_host_key.pub;
}
# rec {
# idx = 210;
# name = "fs-work";
# address = "172.16.239.${toString idx}";
# ipoib = "172.16.240.${toString idx}";
# pubkey = ../fs-work/ssh_host_key.pub;
# }
];
etcdCluster = import ../etcdCluster.nix;
name = "rossby-manage";
address = "172.16.239.221";
ipoib = "172.16.240.221";
in {
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
};
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
environment.systemPackages = with pkgs; [
rdma-core
hwloc
headscale
];
cluster = {
k8sNode = true;
compute = false;
slurm = true;
mounts = {
rdma.enable = false;
automount.enable = true;
users = false;
opt = false;
work = false;
data = false;
ceph = false;
};
};
features = {
desktop.enable = false;
cachix.enable = false;
host = {
inherit address;
inherit name;
};
myvnc.enable = false;
os = {
externalInterface = "enp129s0f0";
nfs.enable = false;
nfs.exports = ''
/exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
hpc = {
slurm.server = true;
slurm.slurmrestd = true;
slurm.mungeUid = 993;
manager = true;
};
k8s = {
master.enable = true;
node.enable = true;
nodes = computeNodes;
inherit etcdCluster;
};
monitoring = {
server = {
enable = false;
scrapeHosts = [
"rossby-login"
"rossby-manage"
"fs-work"
] ++ (builtins.map (x: x.name) computeNodes);
defaultAlertReceiver = {
email_configs = [
{ to = "jonas.juselius@oceanbox.io"; }
];
};
pageAlertReceiver = {
webhook_configs = [
{
url = "https://prometheus-msteams.k2.itpartner.no/ekman";
http_config = {
tls_config = { insecure_skip_verify = true; };
};
}
];
};
};
webUI.enable = false;
webUI.acmeEmail = "acme@oceanbox.io";
webUI.allow = [
"10.1.2.0/24"
"172.19.254.0/24"
"172.19.255.0/24"
];
infiniband-exporter = {
enable = true;
nameMap = ''
# 0xe8ebd3030024981e "c0-1"
'';
};
slurm-exporter = {
enable = true;
port = 6080;
};
};
};
programs.singularity.enable = true;
# services.udev.extraRules = ''
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
# '';
services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
services.flannel.iface = "enp129s0f0";
networking = {
useDHCP = false;
hostName = name;
interfaces.enp129s0f0 = {
useDHCP = false;
ipv4.addresses = [ {
inherit address;
prefixLength = 24;
} ];
};
# interfaces.eno2 = {
# useDHCP = false;
# ipv4.addresses = [
# {
# inherit address;
# prefixLength = 24;
# }
# ];
# };
# interfaces.ens2f1np1 = {
# useDHCP = false;
# ipv4.addresses = [
# {
# address = "172.16.239.99";
# prefixLength = 24;
# }
# ];
# };
# interfaces.ibs2f0 = {
# useDHCP = false;
# ipv4.addresses = [
# {
# address = ipoib;
# prefixLength = 24;
# }
# ];
# };
defaultGateway = "172.16.239.1";
firewall = {
allowedTCPPorts = [ 4443 4725 ];
extraCommands = ''
# needed for nodeport access on k1 and k2
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -d 172.16.239.0/24 -j MASQUERADE
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
# iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j SNAT --to-source 10.255.242.3
'';
};
};
fileSystems = {
"/exports/public" = {
device = "/srv/public";
options = [ "bind" ];
};
};
nix.extraOptions = ''
# secret-key-files = /etc/nix/ekman.key
'';
programs.msmtp = {
enable = true;
accounts = {
default = {
auth = false;
tls = false;
tls_starttls = false;
port = 24;
from = "rossby@oceanbox.io";
host = "smtpgw.itpartner.no";
# user = "utvikling";
# password = "S0m3rp0m@de#21!";
};
};
defaults = {
aliases = "/etc/aliases";
};
};
services.prometheus.alertmanager.configuration.global = {
smtp_smarthost = "smtpgw.itpartner.no";
# smtp_auth_username = "utvikling";
# smtp_auth_password = "S0m3rp0m@de#21!";
smtp_hello = "rossby.oceanbox.io";
smtp_from = "noreply@rossby.oceanbox.io";
};
security.pam = {
services.sshd.googleAuthenticator.enable = true;
loginLimits = [
{
domain = "@users";
item = "rss";
type = "hard";
value = 16000000;
}
{
domain = "@users";
item = "cpu";
type = "hard";
value = 180;
}
];
};
system.activationScripts = {
home-permissions.text = ''
chmod 755 /home/olean
chmod 755 /home/frankgaa
chmod 755 /home/jonas
chmod 755 /home/stig
chmod 755 /home/bast
chmod 755 /home/mrtz
chmod 755 /home/avle
chmod 755 /home/simenlk
chmod 755 /home/ole
'';
};
# Use nvd to get package diff before apply
system.activationScripts.system-diff = {
supportsDryActivation = true; # safe: only outputs to stdout
text = ''
export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
if [ -e /run/current-system ]; then
${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
fi
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it
services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
PubkeyAuthOptions verify-required
'';
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
virtualisation.docker.enable = pkgs.lib.mkForce true;
# Configuration for the coordination server for a tailscale network run using headscale.
#
# We can set it up to provide several exit nodes through which traffic can be routed.
#
# Servers can join using this command:
# `tailscale up --login-server net.b0.itpartner.no --accept-dns=false --advertise-exit-node`
#
# with the following config:
#
# service.tailscale = {
# enable = true;
# useRoutingFeatures = "server"; # for exit-node usage
# };
#
# Clients can join using this command:
# `tailscale up --login-server net.b0.itpartner.no --accept-dns=false`
#
# services.headscale = {
# enable = true;
# address = "0.0.0.0";
# port = 4725; # hscl
# settings = import ./headscale/settings.nix;
# };
services.tailscale = {
enable = true;
authKeyFile = "/var/lib/secrets/tailscale.key";
useRoutingFeatures = "both"; # for exit-node usage
extraUpFlags = [
"--login-server=https://headscale.svc.oceanbox.io"
"--accept-dns=true"
"--advertise-exit-node"
"--advertise-routes=172.16.239.0/24,172.16.238.0/24"
];
};
imports = [
./hardware-configuration.nix
../default.nix
../mounts.nix
../myvnc.nix
];
}

44
rossby/manage/hardware-configuration.nix Normal file
View File

@@ -0,0 +1,44 @@
# Do not modify this file! It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations. Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:
{
imports =
[ (modulesPath + "/installer/scan/not-detected.nix")
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ];
boot.initrd.kernelModules = [ ];
boot.kernelModules = [ "kvm-amd" ];
boot.extraModulePackages = [ ];
fileSystems."/" =
{ device = "/dev/disk/by-uuid/08147fd3-5fee-4650-bfbf-43f4e5c8659d";
fsType = "ext4";
};
fileSystems."/boot" =
{ device = "/dev/disk/by-uuid/392A-D2D9";
fsType = "vfat";
options = [ "fmask=0077" "dmask=0077" ];
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
# (the default) this is the recommended approach. When using systemd-networkd it's
# still possible to use this option, but it's recommended to use it in conjunction
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
networking.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f1.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f2.useDHCP = lib.mkDefault true;
# networking.interfaces.enp129s0f3.useDHCP = lib.mkDefault true;
# networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.enp65s0np0.useDHCP = lib.mkDefault true;
# networking.interfaces.tailscale0.useDHCP = lib.mkDefault true;
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
}

1
rossby/manage/ssh_host_key.pub Normal file
View File

@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCdTAf77QvX1WBN6LAIqPOnSSTMiubIHX+uC7usA6Bvgi9NwLk6aTdslxlA4pko8FOI8D0S06dkoUPqR8CZO+qhgyXi25WCaJ+yvVbC/ySdWsy+I8MA9+9G2FLCXXFQCJmmnDu4W6b91Hul/hv7xIxhXQA+aFpm2UoQ0bRZoMKgyTyue5HWTMsQ/mkug1PmferOWIzLb/N6gQienCeyoU2KGoHY8O30vYCS0M6jvLkq11X769bjYuLZrLS+dM+uGTRdO0nMITUJlT+Rbzb5k1bzuTPpcPM0V9LEPkBMUsgCnJZtvdOta5wf87ef3xrrV3JYmdRvoH6c52+1KoKu6JF29R/6MTJYN7UOS2R7Lksjst5SVnW3gL2AtNWvH0Y+IgaxaYut++Rm4tBCTGV14G0pw0B31Q3018Z5tPbt8dlUFi5D+c9mKx7x1W6GKmll8TVz3FZ6CHi1Dv0AJB+3wCUokwcnMvMymj9hCpLgat44qctGcRoVV7vOMxYFq8toGAmszqktNJR4IGUwUTrJahEHmXe5dU6TxkweF6Np8CJ9VD4f/cEFFaxFLWekfj1aNuq6vvtq0cFdhxkivse3WQAT1HzAS/iHczqkqUp+cseics+LsRPMzrmgjWa1067dzlX5XYPcPfXVxAq7O9JGD4W7BBHusXpju1Zotpjchl2U+Q== root@rossby-manage

131
rossby/mounts.nix Normal file
View File

@@ -0,0 +1,131 @@
{ lib, config, ... }:
with lib;
let
cfg = config.cluster.mounts;
options =
[ "soft" "defaults" "vers=4.2" ] ++
(if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []);
users =
if cfg.users then {
"/users" = {
device = "172.16.239.222:/home";
fsType = "nfs4";
options = [
"soft"
"defaults"
"noauto"
"x-systemd.automount"
];
};
} else {};
opt =
let
server = "172.16.239.222";
in
if cfg.opt then {
"/opt/bin" = {
device = "${server}:/opt/bin";
fsType = "nfs4";
inherit options;
};
"/opt/sif" = {
device = "${server}:/opt/sif";
fsType = "nfs4";
inherit options;
};
"/opt/singularity" = {
device = "${server}:/opt/singularity";
fsType = "nfs4";
inherit options;
};
} else {};
data =
if cfg.ceph then {
"/data" = {
device = "/ceph";
options = [ "bind" ];
};
} else {};
work =
if cfg.work then {
"/work" = {
device = "172.16.240.210:/work";
fsType = "nfs4";
options = options ++ (if cfg.rdma.enable then [ "rdma" ] else []);
};
} else {};
ceph =
if cfg.ceph then {
"/ceph" = {
device = "oceanbox@.data=/";
fsType = "ceph";
options = [
"mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789"
"_netdev"
];
};
} else {};
fileSystems = users // opt // data // work // ceph;
automount = mountpoint:
if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then
[{
wantedBy = [ "multi-user.target" ];
automountConfig = {
TimeoutIdleSec = "600";
};
where = mountpoint;
}]
else [];
automounts =
[] ++
automount "/work" ++
automount "/opt" ++
automount "/backup" ++
automount "/data";
cephConf =
if cfg.ceph then {
"ceph/ceph.conf" = {
text = ''
[global]
mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789
log file = /tmp/ceph-$pid.log
[client.oceanbox]
key = replaceme
[client.rbd]
key = replaceme
'';
mode = "0660";
group = "admin";
};
} else {};
in
{
options.cluster.mounts = {
rdma.enable = mkEnableOption "Enable NFS over RDMA";
gbe100.enable = mkEnableOption "Enable NFS over 100 GbE";
automount.enable = mkEnableOption "Enable NFS automounting";
users = mkEnableOption "Enable /users";
opt = mkEnableOption "Enable /opt";
data = mkEnableOption "Enable /data";
work = mkEnableOption "Enable /work";
ceph = mkEnableOption "Enable /ceph";
};
config = {
fileSystems = fileSystems;
environment.etc = cephConf;
systemd.automounts = automounts;
boot.kernelModules = [ "rbd" ];
};
}

2
rossby/munge.key Normal file
View File

@@ -0,0 +1,2 @@
ç£/ik±/¨÷|ñR¯E¥R®$ÃQfj5·<35>rd<0E>С¶7“{¢–99âTÂîÛ›Ãi‹ÄŒ‰–,ÐŒÍhçïÙ8töv:%‘T”
|ÈÚÈ´þΕ§VŒ00w<30>|ŸÏ®÷íà|È_ŸY{3L_!F1TdÔ&F7õ™B°R

44
rossby/myvnc.nix Normal file
View File

@@ -0,0 +1,44 @@
{ pkgs, lib, config, ... }:
with lib;
let
cfg = config.features.myvnc;
myvnc = pkgs.writeScriptBin "myvnc" ''
#!${pkgs.runtimeShell}
uid=`id -u`
port=$((9000+$uid))
shell=`getent passwd $(id -un) | awk -F : '{print $NF}'`
vnc=${pkgs.turbovnc}/bin/vncserver
# vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver
case $1 in
-p|--port) shift; port=$1 ;;
kill|stop)
display=$($vnc -list | sed -n 's/^\(:[0-9]\+\).*/\1/p'| head -1)
$vnc -kill $display
exit 0
;;
esac
ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1
[ $? = 1 ] && $vnc -rfbport $port
echo "Xvnc server is running on port $port."
# exec $shell -i
'';
configuration = {
services.xserver.windowManager.fluxbox.enable = true;
environment.systemPackages = with pkgs; [
alacritty
myvnc
];
};
in {
options.features.myvnc = {
enable = mkEnableOption "Enable mynvc script";
};
config = mkMerge [
(mkIf config.features.myvnc.enable configuration)
];
}

31
rossby/overlays.nix Normal file
View File

@@ -0,0 +1,31 @@
self: super:
let
msmtp = super.msmtp.overrideAttrs (attrs: rec {
configureFlags = attrs.configureFlags ++ [ "--with-tls=openssl" ];
buildInputs = attrs.buildInputs ++ [ super.openssl ];
});
lib = super.lib;
squashfsTools = super.squashfsTools;
cryptsetup = super.cryptsetup;
singularity = super.singularity.overrideAttrs (attrs: rec {
installPhase = ''
runHook preInstall
make -C builddir install LOCALSTATEDIR=$out/var
chmod 755 $out/libexec/singularity/bin/starter-suid
# Explicitly configure paths in the config file
sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf
sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf
sed -i 's|^shared loop devices = no|shared loop devices = yes|' $out/etc/singularity/singularity.conf
runHook postInstall
'';
});
in
{
inherit singularity;
# inherit msmtp;
}