diff --git a/configuration.nix b/configuration.nix deleted file mode 100644 index d7e7bde..0000000 --- a/configuration.nix +++ /dev/null @@ -1,4 +0,0 @@ -_: -{ - imports = [ ./rossby/manage ]; -} diff --git a/ekman/bin/adduser.sh b/ekman/bin/adduser.sh new file mode 100755 index 0000000..107110f --- /dev/null +++ b/ekman/bin/adduser.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +id=$1 +user=$2 +name="$3" + +grp="\ $user = { gid = "$id"; };" + +read -d '' usr << EOF +\\\ $user = {\\\n\ + description = "$name";\\\n\ + home = "/home/$user";\\\n\ + group = "$user";\\\n\ + extraGroups = [\\\n\ + "users"\\\n\ + "docker"\\\n\ + ];\\\n\ + uid = $id;\\\n\ + isNormalUser = true;\\\n\ + createHome = true;\\\n\ + openssh.authorizedKeys.keys = [];\\\n\ + };\\\n\ + +EOF + +sed -i " +/# @grp@/i $grp +/# @usr@/i $usr +" stokes/users.nix diff --git a/ekman/bin/copy-hardware-configuration.sh b/ekman/bin/copy-hardware-configuration.sh new file mode 100755 index 0000000..21505a7 --- /dev/null +++ b/ekman/bin/copy-hardware-configuration.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +if [ $# != 1 ]; then + echo "usage: copy-hardware-configuration.sh name" + exit 1 +fi + +node=$1 + +[ -e $node.nix ] && mv $node.nix $node.nix.bak + +scp root@$node:/etc/nixos/hardware-configuration.nix $node.nix diff --git a/ekman/bin/deploy.sh b/ekman/bin/deploy.sh new file mode 100755 index 0000000..d13f4d4 --- /dev/null +++ b/ekman/bin/deploy.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." + +if [ $# = 0 ]; then + echo "usage: deploy.sh name ..." + exit 1 +fi + +if [ ! -f $TOP/$1/default.nix ]; then + echo "error: $1 does not contain a deployment" + exit 1 +fi + +cd $TOP/$1 + +nixops list | grep -q $1 +if [ $? = 0 ]; then + echo "--- Updating deployment" + nixops modify -d $1 . +else + echo "--- Creating deployment" + nixops create -d $1 . +fi + +echo "--- Deploying $1" +nixops deploy -k -d $* --allow-reboot diff --git a/ekman/bin/initca.sh b/ekman/bin/initca.sh new file mode 100755 index 0000000..0491dd5 --- /dev/null +++ b/ekman/bin/initca.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." + +if [ "x$1" = "x" ]; then + echo "usage: initca.sh {cluster}" + exit 1 +fi + +ca=$TOP/modules/initca.nix + +cd $TOP/$1 + +echo "--- Preparing CA certificate" +nix-build -o ca $ca + +echo "--- Safeguarding CA certificate" +nix-store --add-root $(pwd)/ca --indirect -r $(nix-instantiate --add-root $ca) diff --git a/ekman/bin/reboot.sh b/ekman/bin/reboot.sh new file mode 100755 index 0000000..edd18a6 --- /dev/null +++ b/ekman/bin/reboot.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." + +if [ $# = 0 ]; then + echo "usage: reboot.sh cluster " + exit 1 +fi + +d=$1 +shift +nixops reboot -d $d $* diff --git a/ekman/bin/ssh.sh b/ekman/bin/ssh.sh new file mode 100755 index 0000000..e49368f --- /dev/null +++ b/ekman/bin/ssh.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." + +if [ $# = 0 ]; then + echo "usage: ssh.sh cluster ..." + exit 1 +fi + +d=$1; shift + +nixops ssh-for-each -d $d -- $@ diff --git a/ekman/bin/teardown.sh b/ekman/bin/teardown.sh new file mode 100755 index 0000000..7b167db --- /dev/null +++ b/ekman/bin/teardown.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +TOP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +reboot=no +case $1 in + --reboot) reboot=yes; shift ;; +esac + +if [ $# != 1 ]; then + echo "usage: teardown.sh [--reboot] name" + exit 1 +fi + +d=$1 +tmp=$TOP/.$d.$$ + +teardown () { + mkdir -p $tmp + cp -r $TOP/$d/* $tmp + sed -i '/k8s *= *{/,+1 s/enable *= *true/enable = false/' $tmp/cluster.nix + nixops modify -d $d $tmp + nixops deploy -d $d + [ $reboot = yes ] && nixops reboot -d $d + nixops ssh-for-each -d $d \ + "rm -rf /var/run/kubernetes /var/lib/kubernetes /var/lib/etcd /var/lib/kubelet /var/lib/cfssl" + rm -rf $tmp +} +cat << EOF + +************************************************************************ +*** *** +*** WARNING: This will irrevokably destroy the running cluster! *** +*** *** +************************************************************************ + +EOF + +echo "Are you sure you want to tear down $d? (YES/no)" +read a +case $a in + YES) teardown ;; + *) echo "Bailing out." ;; +esac + diff --git a/ekman/c0/connauthfile b/ekman/c0/connauthfile new file mode 100644 index 0000000..37cd965 --- /dev/null +++ b/ekman/c0/connauthfile @@ -0,0 +1 @@ +äˇq‹u \ No newline at end of file diff --git a/ekman/c0/default.nix b/ekman/c0/default.nix new file mode 100644 index 0000000..e5e0519 --- /dev/null +++ b/ekman/c0/default.nix @@ -0,0 +1,110 @@ +{ pkgs ? import {} }: +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; + # }) {}; + # pkgs = import {}; + + nodes = import ./nodes.nix; + + compute = { + deployment.tags = [ "compute" "c0" ]; + + fileSystems = { + "/users" = { + device = "172.16.239.222:/home"; + fsType = "nfs4"; + options = [ + "soft" + "defaults" + "noauto" + "x-systemd.automount" + ]; + }; + }; + + systemd.automounts = [ + { + where = "/users"; + wantedBy = [ "default.target" ]; + } + ]; + }; + + mkCompute = host: + let +<<<<<<<< HEAD:rossby/c0/default.nix + hw = ./hardware-config.d + "/${host.name}.nix"; +======== + hw = ./hardware-configuration.d + "/${host.name}.nix"; +>>>>>>>> ekman:ekman/c0/default.nix + in { + "${host.name}" = { + cluster = { + compute = true; + k8sNode = true; + mounts = { + rdma.enable = false; + automount.enable = true; + users = true; + opt = true; + work = false; + data = false; + ceph = false; + }; + }; + + features = { + host = { + name = host.name; + address = host.address; + }; + os.externalInterface = "enp65s0f0"; + hpc.compute = true; + # hpc.knem = true; + }; + + deployment.targetHost = host.address; + + # services.udev.extraRules = '' + # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + networking = { + hostName = host.name; + useDHCP = false; + interfaces. enp65s0f0 = { + useDHCP = false; + ipv4.addresses = [ { + address = host.address; + prefixLength = 24; + } ]; + # ipv4.routes = [ { + # address = "10.255.242.0"; + # prefixLength = 24; + # via = "10.255.241.100"; + # } ]; + + }; + # interfaces."ibp1s0.7666" = { + # interfaces."ibp1s0" = { + # useDHCP = false; + # ipv4.addresses = [ { + # address = host.ipoib; + # prefixLength = 24; + # } ]; + # }; + }; + imports = [ + hw + ../default.nix + ../mounts.nix + #./kernel.nix + ]; + } + // compute; +}; +in builtins.foldl' (a: n: a // mkCompute n) {} nodes + diff --git a/ekman/c0/hardware-configuration.d/c0-1.nix b/ekman/c0/hardware-configuration.d/c0-1.nix new file mode 100644 index 0000000..185e1ef --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-1.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/0a0feff4-c3aa-4a67-9880-78c08f2e1db3"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/A52D-B7FA"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-10.nix b/ekman/c0/hardware-configuration.d/c0-10.nix new file mode 100644 index 0000000..88ddfc7 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-10.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/cd6d1033-a92f-4ab7-8265-00d57466221b"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/AEA2-6A41"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-11.nix b/ekman/c0/hardware-configuration.d/c0-11.nix new file mode 100644 index 0000000..23a463a --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-11.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/92a043ce-b2be-4cdd-b6e0-1e8f3e6952fd"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/3038-56D0"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-12.nix b/ekman/c0/hardware-configuration.d/c0-12.nix new file mode 100644 index 0000000..d6e6e1b --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-12.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/5d8be7db-bdb6-47d2-bb9a-5dc62f162c0f"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/8C14-D452"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-13.nix b/ekman/c0/hardware-configuration.d/c0-13.nix new file mode 100644 index 0000000..19898f1 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-13.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f0909558-f71a-472e-af38-8ffdb2e9e453"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/72A0-CF21"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-14.nix b/ekman/c0/hardware-configuration.d/c0-14.nix new file mode 100644 index 0000000..4527c39 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-14.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/c35b6b6f-29da-4ea0-8fba-067301ce9572"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/3FC3-84E2"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-15.nix b/ekman/c0/hardware-configuration.d/c0-15.nix new file mode 100644 index 0000000..693bf56 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-15.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/1a765ce5-2156-4427-8594-824473a31205"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/4CD7-6D3E"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-16.nix b/ekman/c0/hardware-configuration.d/c0-16.nix new file mode 100644 index 0000000..7323c07 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-16.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/2580dfeb-a28e-4f3d-87a8-e7c7a9fdfeca"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/6BB5-3DC2"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-2.nix b/ekman/c0/hardware-configuration.d/c0-2.nix new file mode 100644 index 0000000..270ddb7 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-2.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/032fe2e6-9091-47e8-b121-0123abe4f0fc"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/EE11-AD60"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-3.nix b/ekman/c0/hardware-configuration.d/c0-3.nix new file mode 100644 index 0000000..eaf215b --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-3.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/da170cd8-0a6b-470f-96aa-a7bdf3dd8736"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/159D-E4C1"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-4.nix b/ekman/c0/hardware-configuration.d/c0-4.nix new file mode 100644 index 0000000..3ec10fa --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-4.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/b081d8a1-3cae-4017-afd3-0482d58f3062"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/1E63-EA5E"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-5.nix b/ekman/c0/hardware-configuration.d/c0-5.nix new file mode 100644 index 0000000..0cae6d1 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-5.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/494bceb1-c0d1-419d-a379-8104ff90aa61"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/294F-20BD"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-6.nix b/ekman/c0/hardware-configuration.d/c0-6.nix new file mode 100644 index 0000000..4df363a --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-6.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/be0a7204-1a90-4f06-980a-a315e9c664ca"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/4889-6E82"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-7.nix b/ekman/c0/hardware-configuration.d/c0-7.nix new file mode 100644 index 0000000..3514f6a --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-7.nix @@ -0,0 +1,37 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f972da1f-a7da-4c94-8bee-cf733a3e4aef"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/7031-4731"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-8.nix b/ekman/c0/hardware-configuration.d/c0-8.nix new file mode 100644 index 0000000..310c276 --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-8.nix @@ -0,0 +1,43 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/ffeb8ede-16f1-48bc-a976-1ea9ab982745"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/E780-1C40"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/hardware-configuration.d/c0-9.nix b/ekman/c0/hardware-configuration.d/c0-9.nix new file mode 100644 index 0000000..c91b78f --- /dev/null +++ b/ekman/c0/hardware-configuration.d/c0-9.nix @@ -0,0 +1,44 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "mpt3sas" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/3d27bbdb-4370-4af2-9747-74d03acfa558"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/9E1B-2495"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp34s0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.tailscale0.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0/kernel.nix b/ekman/c0/kernel.nix new file mode 100644 index 0000000..6aaa81c --- /dev/null +++ b/ekman/c0/kernel.nix @@ -0,0 +1,58 @@ +{pkgs, lib, ...}: +let + kernel = pkgs.linuxPackages.kernel; + i40e = + pkgs.stdenv.mkDerivation rec { + name = "i40e-${version}-${kernel.version}"; + version = "2.13.10"; + + src = pkgs.fetchFromGitHub { + owner = "dmarion"; + repo = "i40e"; + rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd"; + sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr"; + }; + + hardeningDisable = [ "pic" ]; + + configurePhase = '' + cd src + kernel_version=${kernel.modDirVersion} + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk + export makeFlags="BUILD_KERNEL=$kernel_version" + ''; + + installPhase = '' + install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko" + ''; + + dontStrip = true; + + enableParallelBuilding = true; + + meta = { + description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)"; + homepage = https://github.com/dmarion/i40e; + license = lib.licenses.gpl2; + }; + }; +in +{ + # i40e2 = i40e; + # boot.kernelPackages = pkgs.linuxPackagesFor (pkgs.linux_5_10.override { + # argsOverride = rec { + # src = pkgs.fetchurl { + # url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz"; + # sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid"; + # }; + # version = "5.10.239"; + # modDirVersion = "5.10.239"; + # }; + # }); + boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10; + # overlay = self: super: { + # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; + # }; +} + diff --git a/ekman/c0/nodes.nix b/ekman/c0/nodes.nix new file mode 100644 index 0000000..44cc1bc --- /dev/null +++ b/ekman/c0/nodes.nix @@ -0,0 +1,11 @@ +with builtins; +let + nodes = genList (n: n + 1) 16; in +map (n: ( + rec { + idx = 100 + n; + name = "c0-${toString n}"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ./. + "/ssh_host_key.d/c0-${toString n}.pub"; + })) nodes diff --git a/ekman/c0/ssh_host_key.d/c0-1.pub b/ekman/c0/ssh_host_key.d/c0-1.pub new file mode 100644 index 0000000..ae87018 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-1.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKMILMKLqs/kq0ToCFJRRIaJFPh0FP0IHGjrg1G+UXB5aepVQ0GgiFeG9RiAtXbCNEqJDTXfhRRY2FJoNOghFEZ0Z6ECfbvS79OT8XDu2iouMHmu3Xtbh0LGNdoV3LHoSOrcUZnQwQXeQu5J12AIbyC5ng3vOf3Ud9pVehM7WqRegSiaqacneISfq1JtHzWcO7P/Rd9DtyEuFDuTZHLSPrF4+bnbLW7WIgPGTzs28PvWurhmZg01ahcadACAeDmxVFHvSjZbNJbFMPVrNkJbNzXyw8v2WN8wISmiMbJ+ANi/zBgiyc+TOgnbb3us5m6+qkhK7MboQz22y3aFFb/hUENk5aQEpyp7zG5Ua80eGue3EZ5iLselHI7J6w3UUST0tzY7J5uwhKI6Vx6m70s5qUZWG0oF06Y/+xC6GNu5+JBDd7E7l/TjXTtSmlVXcUOjhqK+qtwOsNtFKdWGthdEKuNoa4sfv7tkmtXIvjBuceUMgaHB0hQYm34cj5k82GXdg8U5xkf7dQkv0M2VrPJ2f8KvjSlVM3YvMQ5Pb6WaB5kP0PGEsDY0X5OHMwc6UhYhngJo1sIlwYQwXVQtjj0wVO4jEuGZczgq3FqusHXbD8uOy3eQxLIZOebvzhAnzaDfXhyIM8jhG9WHQMprJf+No+LlisPEBOlnHcxylsCV8VGQ== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-10.pub b/ekman/c0/ssh_host_key.d/c0-10.pub new file mode 100644 index 0000000..10f0c64 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-10.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVoWZnPzLuLfiTH82RIAweOqGfdZvKJe2LkaD39rPFr8w5gEIIYmorquYndpSynMH9P4N/HZZ0/kvyjFps4i6uZrJ6gBIZkFCayN2ljyf4/7kvIGlSQQ6c7a1Bty9oFIZYbsHFNNbCUqu8IF1aT3PBoiAjLZFkQlAaDUanIHb34XgDvaweU+s88QG0q7f7MN8eUlBkY4l34pDS2P28xfhbUl5xkDU5/cQXwZxdETsLgwPweXuCDQ58ZqYwqHUrp7GNfJ5QF9CnqTClrRKxfxJeUcDkvBnTWAvIAzMheWouBypyEguuaZDmvp80ylNKxkin18LFC61wOVHK6/1tDRp45CJ3Vhb9T+ch0ys0k3Qg/ebyCkKx1mOuV6waMq7imjWN4sQh7p6BjbrHuxFilczfv87F8p7TlzBMGDh8+ffL7G0ude578kEf05dHZT1M5o859l7iANgPYgEeaAocnZhwWy/0AYSG/zZeEVNCRS7YO0mTOwzPNmf1DhxArX9DrVPODwc7iHRVEy4KnqW+XcP8TcLQXOlqGEIIa+iE/AJlm2siqreoaUNsl9Gh48aiTX/90uzDnnIsMkNRDD05h0JwI1I7itAcTJVs6BX68CyIhD0VbNILsFqTGOHnJWiBR8z+79XnZwheZPM7xhv0n1mTvmi3syB8kZq6sGL6q7zTmw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-11.pub b/ekman/c0/ssh_host_key.d/c0-11.pub new file mode 100644 index 0000000..dc6705c --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-11.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSuaHY3hT//NqibrlVTpe4rBOixEVENAS4XG7cfM0pGZlVeRH1ebTki849pdpQ93j+r4Am4kTLYpIq5RHL+Pzpvp7FlY6DgeS/OCtyXNIC3gGe9xJU2khxRWkbYy7PXnjcFY19djy9Tkdgd9oIJqGYc190t9PPnwhG2TGo6LT6t0kTh0LEC3culgU9EAFAHkPaqnGjs5ElRknOqZ1M3jzLMKuL3+ScKRk33q+jb0+Gw/4Gl9fv/BPWQclP7E4PQaddc9oe/SqMN2lXMVV93n+QW/bImBtCAT6873h6wjDlKUmXu2WZPafiDjavbKLpgMuUSXzq9COxHGYN7cbqJENskzxz0awvRE5cBOL85qfZwkdN2uaKbdjRWp+EEcAWdLxWX/QlUS9PruQL1bVNe956nzyqsF3BH1f0M/nLuOv3ArJ0qGS1HgjJT5doVAUl76TqyNy37WZdIDVBFtsmz9XKZfei/1QzY0CKAFw8ZLKb+w68b19A8zrFNwH1W+lWZwxFGmlmn7kj0aoWYitQiDQ74eu0EJDtHgJHvYIgm4gmkXVLwMPPmg+GYJ1pVvh3ychWsZrHOoIEigZP8t/6vAZSFI7DWbMTXfn/aYSuM1qHhe1L2SqqjfdPwbf0EZjlUmj7C5JPv3a3QsGmUPdeugOLs3ow14iWH9k/NRTYzEUmDQ== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-12.pub b/ekman/c0/ssh_host_key.d/c0-12.pub new file mode 100644 index 0000000..0808800 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-12.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCkogj7O965v6DYNCld411vyUpsAL8+6hbVT2fjC1RZVyR01AW/+Mvle6aM90XpCMgwUCxgZIULNG4waFlCxXt5IGbOO3/vkHdznK9TUNBJllg69KAdp9nAePyZIczzxRXGh5lzu9RAE7Dywz8hkgELwHokGq8BBQ620/prMdlj4GRsSk3A3hdvbC5XuhwL+KJjrSecELjcvDtiURKDgkDglg7b3bahVais0meaMjkfYL4w1nm4AHnwDlriP6y3hJwBH3odJ1MdJmZBnS0Ebf/wYl2vL0kNACaQGh1+gzdhw2zviv7rW9Wuir07h6CuAQ1SV/b92lzk2Zekaeixf/6I1YNHWiQhfEVOZG/zibm8LdidJxUoLCxyO0h2nBGI3iuPgzUzAmrDSG8nSGNMJw3GU6ub4nj8s+6ZxV7nZ8omozqm6CP9Dw4H4iSRF1CVS3ePH44Zssgd7sbaPG5w4NAniKLRiTTwFfLYyyAhm7APHNrunFPX9SD2H2rJiqJ+16e4rDiBFjoQz5AO/+za+ITSNWZX7BLZM2Swo7jvrZWXYQQGhRj9r3Lzw6zZol+waC+uU4i5JBvQk4ksSNWkTAl/i2GpiUdy94ksBRSZCUWA6gQ49M8iftFNsLsT4KkuIpgV06zHSHKs/9NNovwiINiQo+zGBNNy0eGZTOXHIV/6nw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-13.pub b/ekman/c0/ssh_host_key.d/c0-13.pub new file mode 100644 index 0000000..8ba0eb6 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-13.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4wc7cxQGddBOqEbYRIo61cOsil//rkK2OKcuBEVXm4MlkYc7ZZIdSs34YAloPnuioTs1csnsGCGkUA1z4zLHIs/o/wLn1pTmA3ZYVevdaFl6CCDBgO/Tv/KXr/g3hq3Dhv9+2mzbu0Zc+44GBH+NrP77DsJAQiEkDH31U8MzqjMb0pzDbF2qI6GBa0IQPORevbJm3MT61MXvxBResYFDZhmgxA6HUokiUsUm0JImgDctwKAYZyzB9p19rZ9aPJxtnvQzh0EuBQc0Cfi8mlJ08DHFVK1F8/iQ98j0eOkTQuA+cqUBc2r0S5Ea0rK++qfEj0KsZy1av9nYlirUg9xNaWd9m/gMf9uMqY2eK3ly3+eSap8u91XIqm9NjFnLhjw7MXUmA+rXzU5gyfpcBu5pZwRQIMs7y5P42sll/kOhg9/VOmdnQqecvUm2ro10fQS8nRthLwVwHNsPFr5JSe36aKbF4bQIYZCBPmz8nLs5THdX3ONREfBRk3T0m5Tm19RvxWSyuaxJlvpTguZ25xk5J+1Qw6jocSsg75u+lP0X/Ty9Q7yCoEbNnLiQXpIU3IAl7RcLk9Edzj4xcpt5QLn6yOdwjn0qrYzke5lClX0l9btixYCyT6r6Z6NvKwJQBJHSDG/I74Brwt5Tr1SRpI6EvywF3rKKL3m9eKVGlrrNt6Q== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-14.pub b/ekman/c0/ssh_host_key.d/c0-14.pub new file mode 100644 index 0000000..cef05ba --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-14.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEniX5RkR0XcPBj+hZd0sC/1RdQhD/P6rvET55XWmGfkfnfhkMi5vHCv976dTk+H/oee1l/J2cfl6VKQpWGneiiH4ZRD6nqM7Du9AChWYFqP05x4vMag6RYO84qWbIKOE0sJYe52c2Q1TA3bZhaZZIojww/z4En1QDGsM3Gt0NqbyiLKlp5C3AVB+mF47vVb2eR32cnYmwa1Ab0ylQeLnVDtMg2QdXBpAQk5HbwXpyfzmH7VJnKVmF1cGei8LaeXB0M/EbbZMQJCXjrBpPQS3JfMPyFYfd7ok3vVvQiukkzRX+1Y7rWbDw20DHC7cQxFQLUU0cwSU2H6Kl+jFbMA40aQw9cZqUIet500p7SGK/32bZJxcV25XLWFBTAVD96mJhK3WPzeubI8tPmFI0Me7H9AMFgO51vcxqqVFzynF393k+DxGsMewWPJyrnar609j2CLRYjLiEDOjRD8Gywwo5QtIxmCMlaJhAzaJVr125Cp4JIxzKFqPfDsi9SmLfdLMlczQjQAvF0kS55laMXBxgM7ZHX1ihKq6bOnx4Lc1cKTeiLVH84srkhmHgY/G+m/eryQaY7feEVmqwbQovwEnCb+trhttibnOIilbTTVH71qE1rZL+hzgeUwL7jheCPzUQAWzdPvLLDYJq0yjGugHljRgtq7q1zFqrLGIaEgGjcw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-15.pub b/ekman/c0/ssh_host_key.d/c0-15.pub new file mode 100644 index 0000000..b012c4e --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-15.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+VIsgEyusSjMuU26gQAYNoWs9iC1ErHVjmZNpbtzZ5IlR4RdsmsStf1QZUSG5FJRA2pMOaZD0QUOFI4t4k4VWsYwVXhgkcjF1u0+nlc8XMSihmIHsNPZNYBVRyuL4shegjezKn4EZoHn41UxAcpWAg9XDIRsENAm1G8RcPqwmhrLg4KpC482iuqJT+GavdEcV5rT0O3wCeq0EyFpyd8wX3JARKiyBA5HjyiVzJTeycda3ffudwDmsym4JQeYHretj5REc1Wn4qOVrn6m/L5K7ttysTXvF3aQOegxkOfFhFvqPYJm/I0J5e2U4yRbwaaTyG2acfdIFLu/wwJUEjN7iQ9pj+uCRqfPEtZj1xe6CxXYBQk932gq2H+Y9QsC898Ubq1G9js+AaT3vSIUenGzhGYZl4VIVWjN6h5JvyxAUAbhBSD3zvV1bx/vv1k4e9l7vk3HyZncEZyL81gIB49VKUkWtef9Z/+5778xFjYrz5ykqdkCZEPa6IpphQz9P59MB64rqfK6VNLHN0LwiGxaLP+ctnx6FLXxVYgUtGQS4bbzPhYsqht1FXx3DIecGye4M2l2MorWvaycmwIp0B+OV6sOdcUc/o13ahFour3764lK7C/fYFe2pwRe5UleJEOd+j8lI8ImLgwM9Aclie4v3mjAOAmaxW1m668X9dOtuyQ== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-16.pub b/ekman/c0/ssh_host_key.d/c0-16.pub new file mode 100644 index 0000000..228f083 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-16.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClq1v4N8cIxay8hXNOwU4ulrnf7Yyvnyt5reijIfOUfMiGskgi//kAdf5LyaQjyoZAApABTATVU/vIKhc/D2x96h1VJc1j+i9UmlUuawf7r5d5vcfar8Mc5T9cvXHOs/AWsjlH3vp5Kxac/A1+uk5VCGcML71iBRov3bZaM8RT2s+hzt5hu0RvclLPYuzPafKdsMJQg7dw8Y6pb7R45+8729ItKMHcK54IUxiOjLMbNCkHyUy6m0pPBTfj8G2PnoFkFdg77O4lVQnptyFU9wxSa76RLEqQp1CftaNPzxF6hD3VgGjIKSuiRz30A7J37lWMur6H5185NjWefDUyADa8JY5blb0BAP+csOrGxv39Ghh0wchE9I6Lozb+wGKphk4Esbq0wtKCsph3AZptyO3WCZeajQK5sZCawjOZUY0/t0cvtDefreDg0Db/cw1M6vc1fDgfIrHQ7Yq+y9QQmrClW+1b6pcv5DBKiipa9fUOZNkFcW+do5RKqr52IkuhGcG0pb5mbiGSBIb5EqgoEqBi2o5E5RQ8nWf9tF3xCrksCOjAnu5uUUoxPi14NC7z7p6fmwD3xvA/K3BZYkz1SrsccDcpNCSagPvxzdNTfBiW0USIMmWLQQ0lW7j8GjMGK6B0usNApj1WkiSHuzHNMQLVARaZGANXi/T8JfQoglglzw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-2.pub b/ekman/c0/ssh_host_key.d/c0-2.pub new file mode 100644 index 0000000..2822ed3 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-2.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwC5iT7ZRgapwdzsgmFEkOETikSjDAX/z5peDh5G5rKSoakmSqde3tRJ5fRk1vptW/LkToZT+wU7F6PBQRuI/OyOBIud5Vubr+lqctHy2AfvI1x8SdMRPAIpAhklISKsV9iQH+oMgl7ND4gfLblBbLksQ8iCojlsXcoTwjlt8HYnGNJV/kC5leIIibKDzyMjC0F0jlMiWbK07WllmT0ecNssSV5HNVbY29lW9Fv6J25gp0i025lpB9GC0PiIOoxMVzYRNJCEi1nNPNipuNFOx3XnwRjbVvM1KEP4JqavgXVFXUr2at0C0x+hcKtQ7LpdyvmmAywTMTTljhdU/0jNw8pk57HhpJ1rXQZm4SIqL7gdwDy91KLM3qE1MjzocWZw4YBNDfeNSS6qalxowuut354x8UItEjIwy1uofRF8RKB7LQYqKbXHWCZeA+3rNA4E+QIuvbwX8HObMS9JXzTUSUL7r53rAFNVifYSey6u2iZ+PT+jKkXKOCOE6fuCy/MPVaiJVwtd0wVsgC4HH6CvrJf1dwlI5Av4B8LrnZNgpjog+aqgDHjRRY+3ys5Iwt+DqHHLGiHyoM/mbF0Hl5GQ3SLdfhn27zLndLnBg6uBSIRmssn0Vech4gndy4nUNZpWyKWoy4lt5aVLOoojD9G3Kp5XfQVOlJToU9D73n01hclw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-3.pub b/ekman/c0/ssh_host_key.d/c0-3.pub new file mode 100644 index 0000000..76387a8 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-3.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+rokHymxlz8S5jYU6Vamp1TVX+M/rAHGflMWwpAYND7WSt1LkCICHbgAWx9lCXGDEidMuajENtu62tW2zar3cXRrk/p5SdwCb+NiztQ4uGTzGIMxYA52Cjy6KyMi7pG9C6/91WyHFy9aKPFjbA1c2FPgf7eqIYg9B3uBu75yCWCuZ3ZRs+al3R8Tbqo4J7sSanxWRpdc0t7eU0U/rMeTi41SXmNF82smHaQ7iOnWajHAWwybmNpWQkiFpvyahzct0No606oODFl/SttI8R9CCJpvlEeT/Q+nj0hgMciw3J604EJaaLgaYfUcJg/V4F3fe8eGDiOzBYBYrYguNv/XVUW0llP2K+SG0zi/3luQTMghpd+fsGr49cdwFrJWXJhqvE2O5QlQ3mWw+pwlUY7Qgs2usGbQK04VGpJlFiSkso4pythuNMFB37YZoA57+RozsjpeiVmL37xcq6jfsd7ljvjbUICCGLaaJN7p1c3OQSw1pma1R3G14FvefhUQe+F2DLyLfvFq476KN0TSUU42tw9iKfCbP3geZp7UkqpyONerViP9VOa4CQclyuaZ5Yg1x4v7zmfQ6J8J1+sqSwEUe7UnaPuKM2XklViT0AUy33EKIkzGe91aBOwcQMaA7fYHYf1Jvf5lYI5MiA56hkV8f9KVZsKGD9advK+WnsdyUCQ== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-4.pub b/ekman/c0/ssh_host_key.d/c0-4.pub new file mode 100644 index 0000000..be54298 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-4.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCmlNP7HKPGYtciCLRL6KyPs1JPF/ehYWuMmeiJrVxdq3YSrfAddB/6cdHMOw/RLukdyLdIwvRfGhy6QZq9BbKBFSJKypBGNe3KTTT7zd/qxNKEbQhJ8m43NtBbOUHBPQ49zE2eunGwMYKL2EW/7ndbtzQOTTrpuZTsGXB6W6Rn/ZnYZC9wXmOyn5XJKleOhAlor0wIyORdAPwdY0s9LdJ6lJUCUqxstFecuqIsgnaMUBbzyTl09e2RbtMduEjRH/B7CHBxj8P4oVlBGTVStOAeeqMudrJeXvRssMB1BX9Tu92CVYnr2FsUazJ+s1R0dHqdXFuWuR2oddw3y1Et0RyAoNmgiku+ZwIbqousNZx8UxrGlMII1fi8RdNqmMZE+8LhC3yhztYQloZ8Ak8J8SpQE7av1gvlHKCMD5bxtBD4mNfj3SSdH9TnF8gu1x6zT0A4LdL+PqK6E+2zZlc54DfLT3SwS214kFUtgM4t9VhSC2E+EHGCgZ91U6xmsd4zVGMaGvzRXcyH0rROzCKsP2M6rVssOER1ePXEZUIC+62PC9IY9tM8lFDH2AB/JF/xdLZYtO9pdWG6d6DOKSpiP2qlMILIX9/qivHTCySnyP3lBtJ7THF8zsJ5sTUO53/Grd0mSLSqTm3T12PIwABPWwiEXKMD2pIszUhu44KRAF0sJQ== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-5.pub b/ekman/c0/ssh_host_key.d/c0-5.pub new file mode 100644 index 0000000..2b8a282 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-5.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDmF23dxQHzmN1rVlDkHelCBK48NFc646LFZyB5Gn8cphpur9wV26iOxWOXct+ITU9p6fWmXDko9zCrCWcsF6Ib3rBzbCHKMzF6Vuj0UZg3XP4IUrOdw+Cd5L8GTVoAzskXL0EM+upKPeXFBob8OS3F9Ax0X9ZT66vqY5QGuZWtUikWnakTxbl7JFj+Hgx5/ZzS5cH2e/9NoKkl7wGSkUp0rqjfIEimXI6SNl98kJUtCWEWyUvon4yLuJ4X3+4wrjwdgznspOWgCt0XnTnmak5ADrbz+VuXnDR8TB4AJNfhFySW6E4ZRSrGpZrt1FfaUBxTCmq2Zq/t1chA0Z6z+/k2vCjd9CXodAUalgVmJZ7EQZdSSo8nXA4P5Y8I6TMesTKlBVxMlxqsQfwve+xaPV0/+QSXCvpdRep3+QDVZXaOO4N7UU2xeQR38cNHNzovMy1z5439nq4p5uoA3CxWLL/z6vsZQ3s3xXdvlvPudGWmTW2c0FGKS8ALSWUKLibvPHsEEdIwLJ6FtUs0Vp3QtBKdq/NmN05gZJEOrnfKr8JpvVl9vzRAc8wCYsTyujHTkkYb/E+fb7VsEngbD8iu6PiRgeOui/t0QkTl1ij+SD0YnDXrh61YhQjuvdogJq47gtcSbCE8Q2rkzDpDIEOJNZP7uZINyl9VIRhBb6jt7CC/kw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-6.pub b/ekman/c0/ssh_host_key.d/c0-6.pub new file mode 100644 index 0000000..9c86b14 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-6.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBcImgez4RkX1TuqDTk/AUugMqSfsIjdWmEyJRgGw0ZqYSiz9R0yHVM9E6tFIXkGw73RsF1qVQorcw3ZRrxiYLtJHRyBnX0/6IHFwT5w1TezH7LMJmId3ys+LEcpqnCdVaPo2RVsq3RJhnM4dImVC36UUJj8TiOzhZ03lTP7TCPABxvwzn//+T5EjyKuK7acFU4ts4ii7B28dWErCVjsq29uWxKXopfER+45DjhiilQgJ5R15voBNvvFCR4bfnVrsyBwHXXRohAOU1yJYpsUDkXMDY2DIG41LqpP8lg8IyPF36Vwt4w8vmK6rpxBvy4SKg4TIoxUK3eqODnrgN15/2yDTRWWF8u6rpEopWRQEUlKqZvHnXFWuSN5TGUBeOrjqom92udfGrW0l6Yn6iUEt2n+OkPxRuDAGGwjRf4oV2RSqbhbIKfctJg0b+8VhrecNow2zyNlRvfLiQvKotQEDolvfnOYyIWujsjUWttl5a0eoJBm40QgNvWHJWfuaheMhVfEWcfoqgkvZDfBeE11Z/1eogCXMWELJaX/gevE7I+0veu8myzmRwXQOEbk+Ya8tVCSm6YrZjp7flUDRNMW9bZswEdS/ZjvIlK5RgDCAakBoU4MY/PZipYujFoT4AQTNiV3AfuakktXY8dORUHDOcD8zmdbE6B/6t7d1SrHIOow== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-7.pub b/ekman/c0/ssh_host_key.d/c0-7.pub new file mode 100644 index 0000000..62a162b --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-7.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVFD6R1Mwz0+SqojCUcwLlOm6fKarAPEOB1Ysjo8daGmwx/zfzeHLkzl2RpA8NpSAHheBQJxt+x108wZRZjy+EWYzc/sOaui7lFvZVxxB2ewv1yAoBXUl82O1D2i2ZaaAoLa4G6dZnSfdd0THJNqMIoCCThDaNI3jGcTOzHivVb1XWpdM1DIQtbTcwOJ3Q0UZTiNi5eL+7wlpIKE6/IzkSyTgyYi463CEeR8YXCaILdRweJf5lFem+ahD6uUBe8yuNktpnE8wzQXSHBlNrI3C4Kk9uQicDr1uThexBX8/37hg1V7Wo/AxBQcWlumUCgtnlC53V63XvcRjfB9zrQSbHgZcDnlgGgvMPYillLNCGcFdny1ZbpWpQwlt6kZgHmB3v5m+i63iaCYaAzAoGfp/M/f3mrrQF9GYdsFtngZsGsTxv1Rr9Nmef05V3CewKPaLw13XwPbH+Muvzdx00P9W7P2VmryQXYmAVs4iczUIdRGaTDGJIou4qycRHMAfC6aYEvR2s0+oTzLCMdYgjkVDHxJDkrgSP0Z8++bJa2qPASrTDnT7Hpghup7l1NspJpiZX3vIBLOi+0cBfoRZuLrQ45dDCrRdCySDvv5PfngACqNWt8PCHBeQAHS81d7+xb0e92VfxtZEABTwHlWePMuRmSiSXBi/UGdruFsFm0n0qPQ== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-8.pub b/ekman/c0/ssh_host_key.d/c0-8.pub new file mode 100644 index 0000000..903cde8 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-8.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4dNgyC5n1PQ9hp+o1HuTVt7MXkt5Y7wOFAmR+dzWWQ5X/3p7umJzl/I0955CuaXPTb6E6Oqi8WQAlqlBhk9w8qoqoczXsHbqSbnoWSk82VgvHxPv2+5iWvth/FvuTEiM43tt2AhDxG/8TobR2eFD2/2h/T7OpJUcZ7aHT6BZnN5IS/P6MgarOUootDNx9zn3BXFM4Pg8lnB9dnKm4iO7Ju9kk9s2eWzvk37QpNE+H/h58DOMExUcrDzftXNrVbUeZjw7Ho9MER+fdbucrcdKxFH/NiwXlgRSRd+Z5Y+IG+WJdkdJw1hW47MsGRGH+yEq/woE4YvzdcqqqLh9m2a2ucQkk1pe3xpYXCxwS8aVcIDzlja7rGaa2nSBz+Lkp+G5awT2Sg4ob7vQYIN2X6p5zYsqLfWdU5GloWgjEB/zvADjF7qKeEdFFv/eXzaiwnXSoOofcCZTW0qAldK7cdO/esDUfQGIsti8qk1adXUGXG52UhV0p+rMtloW2AAJVFsUiUWvp2Ue3i+p+nAC17aDMOq1UrR60/n1FcjM7mbwAjLgwQ40tLjc54HoMVg92lMTVgJgk+gBYKFsfrI3nbSmvUiRFZDWzPFSno9WtFasWrNCiQ9srmfP0Ud6ro0QCtnvSxFMuXiVufaUlRka0SLlZbA8N3mLrZ0M+7rWy5syLSw== root@nixos diff --git a/ekman/c0/ssh_host_key.d/c0-9.pub b/ekman/c0/ssh_host_key.d/c0-9.pub new file mode 100644 index 0000000..2a2edd6 --- /dev/null +++ b/ekman/c0/ssh_host_key.d/c0-9.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC9X4NFHsCryXjQOiUrxcTNOx0JpPiGViEuvEhB23C0270bwbxlijaIsTrAxB8lfOaKHGvzXfMBX0lV9QGXfIPTXIr0hbmOy6xfJq6lRQwbiRA6YS4T1yic25mx47F0ODqaW3mTykpIE1qpz4Wzl3M1ZLbbT1wcvWTFJ6MCY74OR1vNrtR7FgVFrh7q7iT1+a8Qzr5zG39yphO9aYiBKNLle820oH1UxsJ3W5e38ZI/huJ2w1Y0TY6eghedEZvGIhm0qDJHlc8EmGterCUjCLZKcZgHNGIC6FnT3n8HGwsTtYHc2mRW3W4BK1iIpncYkDSX71xY4ev13Y7UbD6rVBWqC+eKD/3pm+HlvXxk8tvDZfcq4ngSEIoU7d5FeLgE2hidE92zLaKdXpJ0PaLo1Gz3M0CY3TeaCWZBSnToGY106ynUipoGJ29cDaQWVlYkWAdC7d+3eR1yjpT/98PNx08m8fVs/PT30/LkR6kpUpCB1GBdOu1/Q4DFeGtY1BFNibzoWyk6ByagOMdEfRzPn99J4Vd0zlzwyzFkJTlI7OHeVwBFPHW0joGm3x3MGbRmFQBBIA3C9b9a6mFseQpMX/T++v0h98x7mvsMx2Nf+1O3uRx3yLp47GfyatsKB8oUSi5dI7S3cAyAZIB8E8ot4dkdounPzA2L8ipnzwdEjpqwLw== root@nixos diff --git a/ekman/c0x/connauthfile b/ekman/c0x/connauthfile new file mode 100644 index 0000000..37cd965 --- /dev/null +++ b/ekman/c0x/connauthfile @@ -0,0 +1 @@ +äˇq‹u \ No newline at end of file diff --git a/ekman/c0x/default.nix b/ekman/c0x/default.nix new file mode 100644 index 0000000..c266fcb --- /dev/null +++ b/ekman/c0x/default.nix @@ -0,0 +1,108 @@ +{ pkgs ? import {} }: +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; + # }) {}; + # pkgs = import {}; + + nodes = import ./nodes.nix; + + compute = { + deployment.tags = [ "compute" "c0" ]; + + fileSystems = { + "/frontend" = { + device = "10.255.241.100:/home"; + fsType = "nfs4"; + options = [ + "soft" + "defaults" + "noauto" + "x-systemd.automount" + ]; + }; + }; + + systemd.automounts = [ + { + where = "/frontend"; + wantedBy = [ "default.target" ]; + } + ]; + }; + + mkCompute = host: + let + hw = ./hardware-configuration.d + "/${host.name}.nix"; + in { + "${host.name}" = { + cluster = { + compute = true; + k8sNode = true; + mounts = { + rdma.enable = true; + automount.enable = true; + home = false; + opt = true; + work = true; + data = true; + backup = true; + ceph = false; + }; + }; + + features = { + host = { + name = host.name; + address = host.address; + }; + os.externalInterface = "enp33s0f0np0"; + hpc.compute = true; + hpc.knem = true; + # k8s = { inherit etcdCluster; }; + }; + + deployment.targetHost = host.address; + + # services.udev.extraRules = '' + # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + networking = { + hostName = host.name; + useDHCP = false; + interfaces.enp33s0f3np3 = { + useDHCP = false; + ipv4.addresses = [ { + address = host.address; + prefixLength = 24; + } ]; + ipv4.routes = [ { + address = "10.255.242.0"; + prefixLength = 24; + via = "10.255.241.100"; + } ]; + + }; + # interfaces."ibp1s0.7666" = { + interfaces."ibp1s0" = { + useDHCP = false; + ipv4.addresses = [ { + address = host.ipoib; + prefixLength = 24; + } ]; + }; + }; + imports = [ + hw + ../cluster.nix + ../mounts.nix + # ./kernel.nix + ]; + } + // compute; +}; +in builtins.foldl' (a: n: a // mkCompute n) {} nodes + diff --git a/ekman/c0x/hardware-configuration.d/c0-17.nix b/ekman/c0x/hardware-configuration.d/c0-17.nix new file mode 100644 index 0000000..56810a6 --- /dev/null +++ b/ekman/c0x/hardware-configuration.d/c0-17.nix @@ -0,0 +1,55 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/46dfa481-ccd4-4e4f-b1d7-6875d582f7cd"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/F71C-0BD2"; + fsType = "vfat"; + }; + +# fileSystems."/data" = +# { device = "/dev/disk/by-uuid/125c5d04-5266-43e8-887d-740d5944cb2b"; +# fsType = "xfs"; +# options = [ +# "noatime" +# "nodiratime" +# "logbufs=7" +# "logbsize=255k" +# "largeio" +# "inode63" +# "swalloc" +# "allocsize=131071k" +# ]; +# }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.docker0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0x/hardware-configuration.d/c0-18.nix b/ekman/c0x/hardware-configuration.d/c0-18.nix new file mode 100644 index 0000000..ca38d73 --- /dev/null +++ b/ekman/c0x/hardware-configuration.d/c0-18.nix @@ -0,0 +1,44 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "sd_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/19d00648-5fbd-4464-93a0-e013d7f79d3a"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/68AF-2717"; + fsType = "vfat"; + }; + + fileSystems."/tmp" = + { device = "/dev/disk/by-uuid/bf1bede5-bc60-4603-874c-88ed85e6ab5f"; + fsType = "ext4"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/c0x/kernel.nix b/ekman/c0x/kernel.nix new file mode 100644 index 0000000..17b55e4 --- /dev/null +++ b/ekman/c0x/kernel.nix @@ -0,0 +1,48 @@ +{pkgs, lib, ...}: +let + kernel = pkgs.linuxPackages.kernel; + i40e = + pkgs.stdenv.mkDerivation rec { + name = "i40e-${version}-${kernel.version}"; + version = "2.13.10"; + + src = pkgs.fetchFromGitHub { + owner = "dmarion"; + repo = "i40e"; + rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd"; + sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr"; + }; + + hardeningDisable = [ "pic" ]; + + configurePhase = '' + cd src + kernel_version=${kernel.modDirVersion} + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk + export makeFlags="BUILD_KERNEL=$kernel_version" + ''; + + installPhase = '' + install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko" + ''; + + dontStrip = true; + + enableParallelBuilding = true; + + meta = { + description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)"; + homepage = https://github.com/dmarion/i40e; + license = lib.licenses.gpl2; + }; + }; +in +{ + # i40e2 = i40e; + boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10; + # overlay = self: super: { + # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; + # }; +} + diff --git a/ekman/c0x/nodes.nix b/ekman/c0x/nodes.nix new file mode 100644 index 0000000..fc6f251 --- /dev/null +++ b/ekman/c0x/nodes.nix @@ -0,0 +1,11 @@ +with builtins; +let + nodes = genList (n: n + 17) 2; in +map (n: ( + rec { + idx = 100 + n; + name = "c0-${toString n}"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ./. + "/ssh_host_key.d/c0-${toString n}.pub"; + })) nodes diff --git a/ekman/c0x/ssh_host_key.d/c0-17.pub b/ekman/c0x/ssh_host_key.d/c0-17.pub new file mode 100644 index 0000000..8fa4abf --- /dev/null +++ b/ekman/c0x/ssh_host_key.d/c0-17.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7wnYBwAdcp5+y7QOKzjd2apRv+Nvquy+qr3jVBeiQgbGsjlpGJBRsGv46QhSbB9QLuFuCUluuvilYbS9/ubm+BtRCrFViAHwFx+QhbLYlAXJPLGfXLxJZDSxbM6c9L+N+GAAfhiCfdhUVB39G33v5dd/VD3J4F/xC7lkPJU59pGokqe9dGrDta33pemSzR9QV9RuV4ZxJG1ld1B2L6TjSkAPTb79PPvMw8LnTViNffmHKp5qvwgIfwRvMApG5KnHw720pPncTW4ejY+utSbP4DcwD2qayKbxnsYDpimMDQxR3id6meZa5LcilxExmz/ig7Fpc3TB3wIpT4ha6VVu2FhRThKR5zD1Qif8BaKy5eAmqcWKQm6M9W56LmUdn/OHpzmaSv47UgtJ+1JG4LqGSaShs8kfVWtaqmev6vvnHBZMm7ni3xyROTSWnjJHLDQJbCYb1PBbct2Dod6MuU54CJUuxw0k+hCJ1QMz1ibme3+LauXVuLJ2JrnAoz7x2CcUv9e5MVx0H4HpWTCxi7cNBO5bbeuxMwHbl3JrZ58LBK9zZYl+nr2F7CZqLyoMimQuVpXT6Aw0k/nfYMAXjUJ3qbBldiGsRgpgbvL27nJK0EhQ8Kd36QhHfdJKhiAIPV+06WVWhPllaqay4OgZIQM1aB83eHg/OUxbOWEu89d7w/w== root@nixos diff --git a/ekman/c0x/ssh_host_key.d/c0-18.pub b/ekman/c0x/ssh_host_key.d/c0-18.pub new file mode 100644 index 0000000..908bab9 --- /dev/null +++ b/ekman/c0x/ssh_host_key.d/c0-18.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/a5Q1Y8hta/v8W/cM45o6UimPLfSzz6NM+WaNAYbwnLoz2VVowHdjyndajmH1CMK/cTDRiVSrTkky48f3JNKzCMjmwVA6PjWmRxF0tqcitW9hzFB+L10+fgBsZu7MUMhuWfTtekIgnRjqNZIA3q5PGz9JrMHsRem+RGmDayaNSAkOZ1ToGSS7pnGm06qAswLJaK15wnupw5x9NYO1RXq2Dy2pX68e4urA+h9X9tqDfb6kQP3/9gb+q6BpiGUwUHW5x0HK/LwNqThKV7ZwzgMHqz4rq5TieGemTw8LKcDjj3BWQTDt4SlZ2gux49C7z1vBipHtxKHIdZH9AXYPZCkjgI/OVVJvxkbgMM+3FSr3j/KvTOdL5ycJvGBsF4dJnRTTpay7OmnksQ62+vwNO2dvDnhAgVntrr9Fb1ahLnTN7dOf6BlkUbz+vtYby0gcjr5sW8b4duGpJThDDkAGLUKkx1z94/bpPQpUb1ajXPizO7jjJCK20zTXWG2uiOdXWHmZ3YVoJ/qSOvtUAX2RR2FgJzHqnDnHtmLiMc9OkZ0eJwo6ltBqGHPNXCyBwHAa8eSH2Af+10vfzJ88J8MGD88DV7SYWbJxuB+KMAnDX04UfrvqLnRADzbFuiG8locorXFE90wEZTz/nFWsM+Q/XPA+M1GaO6fK9SyN45LgApKARQ== root@nfs1 diff --git a/ekman/c1/connauthfile b/ekman/c1/connauthfile new file mode 100644 index 0000000..37cd965 --- /dev/null +++ b/ekman/c1/connauthfile @@ -0,0 +1 @@ +äˇq‹u \ No newline at end of file diff --git a/ekman/c1/default.nix b/ekman/c1/default.nix new file mode 100644 index 0000000..85891ea --- /dev/null +++ b/ekman/c1/default.nix @@ -0,0 +1,119 @@ +{ pkgs ? import {} }: +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; + # }) {}; + # pkgs = import {}; + + nodes = import ./nodes.nix; + compute = { + deployment.tags = [ "compute" "c1" ]; + + systemd.automounts = [ + { + where = "/frontend"; + wantedBy = [ "default.target" ]; + } + ]; + }; + + mkCompute = host: + let + hw = ./hardware-configuration.d + "/${host.name}.nix"; + in { + "${host.name}" = { + cluster = { + compute = true; + k8sNode = true; + mounts = { + rdma.enable = false; + gbe100.enable = true; + automount.enable = true; + home = true; + opt = true; + work = true; + data = false; + backup = false; + ceph = true; + }; + }; + + features = { + host = { + name = host.name; + address = host.address; + }; + os.externalInterface = "eno33np0"; + hpc.compute = true; + # k8s = { inherit etcdCluster; }; + }; + + deployment.targetHost = host.target; + + # services.udev.extraRules = '' + # KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + # boot.kernel.sysctl = { + # "net.ipv4.tcp_timestamps" = 0; + # "net.ipv4.tcp_sack" = 1; + # "net.core.netdev_max_backlog" = 250000; + # "net.core.rmem_max" = 4194304; + # "net.core.wmem_max" = 4194304; + # "net.core.rmem_default" = 4194304; + # "net.core.wmem_default" = 4194304; + # "net.core.optmem_max" = 4194304; + # "net.ipv4.tcp_rmem" = "4096 87380 4194304"; + # "net.ipv4.tcp_wmem" = "4096 65536 4194304"; + # "net.ipv4.tcp_low_latency" = 1; + # "net.ipv4.tcp_adv_win_scale" = 1; + # }; + + networking = { + hostName = host.name; + useDHCP = false; + interfaces.eno33np0 = { + useDHCP = false; + ipv4.addresses = [ { + address = host.address; + prefixLength = 24; + } ]; + ipv4.routes = [ { + address = "10.255.242.0"; + prefixLength = 24; + via = "10.255.241.100"; + } ]; + + }; + # interfaces.ibp65s0 = { + # useDHCP = false; + # ipv4.addresses = [ { + # address = host.ipoib; + # prefixLength = 24; + # } ]; + # }; + interfaces.enp65s0np0 = { + useDHCP = false; + ipv4.addresses = [ { + address = host.gbe100; + prefixLength = 24; + } ]; + }; + # firewall.extraCommands = + # if host.name == "c1-1" then '' + # iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE + # '' + # else ""; + }; + imports = [ + hw + ../cluster.nix + ../mounts.nix + ]; + } + // compute; +}; +in builtins.foldl' (a: n: a // mkCompute n) {} nodes + diff --git a/ekman/c1/hardware-configuration.d/c1-1.nix b/ekman/c1/hardware-configuration.d/c1-1.nix new file mode 100644 index 0000000..c97bce4 --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-1.nix @@ -0,0 +1,29 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_7; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/d89e1496-fda1-4de0-b2cc-474967b04402"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/A51A-1F4D"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-2.nix b/ekman/c1/hardware-configuration.d/c1-2.nix new file mode 100644 index 0000000..c34903f --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-2.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/f0826ad5-8a4e-427d-98d3-5afa44440993"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/21A6-D34C"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-3.nix b/ekman/c1/hardware-configuration.d/c1-3.nix new file mode 100644 index 0000000..0179b2c --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-3.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/3af91585-8079-420d-acdf-f60b94d3cfff"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/3590-199A"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-4.nix b/ekman/c1/hardware-configuration.d/c1-4.nix new file mode 100644 index 0000000..c98b526 --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-4.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/84cc9cea-08eb-4b54-8ca3-2aa5c1300a92"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/882D-A342"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-5.nix b/ekman/c1/hardware-configuration.d/c1-5.nix new file mode 100644 index 0000000..ada6f39 --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-5.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/d23386a8-0ca8-4871-b662-decf2b24f4d7"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/AF8A-DEFE"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-6.nix b/ekman/c1/hardware-configuration.d/c1-6.nix new file mode 100644 index 0000000..d4dedcc --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-6.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/2f468316-5832-4684-866d-2e92b08fb68b"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/76FE-F657"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-7.nix b/ekman/c1/hardware-configuration.d/c1-7.nix new file mode 100644 index 0000000..43ad87f --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-7.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/37944fce-07ca-492f-906a-620a37e7e1b3"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/2F51-EC20"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/hardware-configuration.d/c1-8.nix b/ekman/c1/hardware-configuration.d/c1-8.nix new file mode 100644 index 0000000..54107e3 --- /dev/null +++ b/ekman/c1/hardware-configuration.d/c1-8.nix @@ -0,0 +1,28 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/12477966-c6c5-47c6-afdc-35fa7e57e837"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/DB96-7453"; + fsType = "vfat"; + }; + + swapDevices = [ ]; + +} diff --git a/ekman/c1/kernel.nix b/ekman/c1/kernel.nix new file mode 100644 index 0000000..750da0d --- /dev/null +++ b/ekman/c1/kernel.nix @@ -0,0 +1,46 @@ +{pkgs, lib, stdenv, fetchurl, config, kernel ? pkgs.linux, ...}: +let + i40e = + stdenv.mkDerivation rec { + name = "i40e-${version}-${kernel.version}"; + version = "2.13.10"; + + src = pkgs.fetchFromGitHub { + owner = "dmarion"; + repo = "i40e"; + rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd"; + sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr"; + }; + + hardeningDisable = [ "pic" ]; + + configurePhase = '' + cd src + kernel_version=${kernel.modDirVersion} + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile + sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk + export makeFlags="BUILD_KERNEL=$kernel_version" + ''; + + installPhase = '' + install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko" + ''; + + dontStrip = true; + + enableParallelBuilding = true; + + meta = { + description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)"; + homepage = https://github.com/dmarion/i40e; + license = lib.licenses.gpl2; + }; + }; +in +{ + i40e2 = i40e; + overlay = self: super: { + # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; }; + }; +} + diff --git a/ekman/c1/nodes.nix b/ekman/c1/nodes.nix new file mode 100644 index 0000000..d499461 --- /dev/null +++ b/ekman/c1/nodes.nix @@ -0,0 +1,12 @@ +with builtins; +let nodes = genList (n: n + 1) 8; in +map (n: ( + rec { + idx = 120 + n; + name = "c1-${toString n}"; + target = "10.255.241.${toString (idx + 100)}"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + gbe100 = "10.255.244.${toString idx}"; + pubkey = ./. + "/ssh_host_key.d/c1-${toString n}.pub"; + })) nodes diff --git a/ekman/c1/ssh_host_key.d/c1-1.pub b/ekman/c1/ssh_host_key.d/c1-1.pub new file mode 100644 index 0000000..c09679b --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-1.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBEV8wEkeGUOs6umhdeOKYnVlYlta2rOCZSoezvu+bZ4 root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-2.pub b/ekman/c1/ssh_host_key.d/c1-2.pub new file mode 100644 index 0000000..df6aaee --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-2.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDIsqJqWWftoiyiOOHnIuHYqbsaOg4AbKNm80wpjH2E8 root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-3.pub b/ekman/c1/ssh_host_key.d/c1-3.pub new file mode 100644 index 0000000..1351c52 --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-3.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwwu4RJJhKo8s2Mtmpdvxs02d4IwrmS9cTpfAOQ2YkI root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-4.pub b/ekman/c1/ssh_host_key.d/c1-4.pub new file mode 100644 index 0000000..0c58f4b --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-4.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQFQqo2vcj2eYaH7nwdEzgCPme+7g3Db+s16KbQHzLI root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-5.pub b/ekman/c1/ssh_host_key.d/c1-5.pub new file mode 100644 index 0000000..b64a074 --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-5.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImItUKMSygMY/1ZNsyGkyfywyngEDgZ7TxM63UwG1VH root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-6.pub b/ekman/c1/ssh_host_key.d/c1-6.pub new file mode 100644 index 0000000..5c92679 --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-6.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBq+v2bBhQieqfXmtmGYb/9mOv9oc88zerRkkpGpoc0x root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-7.pub b/ekman/c1/ssh_host_key.d/c1-7.pub new file mode 100644 index 0000000..c746bf4 --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-7.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDme/xztp22wQOybtN2TVXMcn2QcVaXtRMp4AnPnzr2T root@nixos diff --git a/ekman/c1/ssh_host_key.d/c1-8.pub b/ekman/c1/ssh_host_key.d/c1-8.pub new file mode 100644 index 0000000..1111820 --- /dev/null +++ b/ekman/c1/ssh_host_key.d/c1-8.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID1WxWnNg+L2+lQ3W2mD0/6kqwuCUOEJImvWnQYLzdUB root@nixos diff --git a/ekman/default.nix b/ekman/default.nix new file mode 100644 index 0000000..a322082 --- /dev/null +++ b/ekman/default.nix @@ -0,0 +1,370 @@ +{ pkgs, lib, config, ... }: +with lib; +let + cfg = config.features.host; + + computeNodes = import ./c0/nodes.nix; + + mkSANs = host: [ + host.name + host.address + "127.0.0.1" + ]; + + configuration = { + system.autoUpgrade.enable = lib.mkForce false; + + nixpkgs.overlays = [ + (import ./overlays.nix) + (import ../modules/overrides/certmgr.nix) + ]; + + boot = { + loader.systemd-boot.enable = true; + loader.efi.canTouchEfiVariables = true; + # kernelPackages = pkgs.linuxKernel.packages.linux_6_9; + kernelModules = [ "ib_umad" "ib_ipoib" "ceph" ]; + # kernelParams = [ + # "console=ttyS0,115200" + # "console=tty0" + # ]; + }; + + console = { + font = "Lat2-Terminus16"; + keyMap = "us"; + }; + + i18n = { + defaultLocale = "en_US.UTF-8"; + extraLocaleSettings = { + LC_CTYPE="en_DK.UTF-8"; + LC_TIME="en_DK.UTF-8"; + LC_PAPER="en_DK.UTF-8"; + LC_NAME="en_DK.UTF-8"; + LC_ADDRESS="en_DK.UTF-8"; + LC_TELEPHONE="en_DK.UTF-8"; + LC_MEASUREMENT="en_DK.UTF-8"; + LC_IDENTIFICATION="en_DK.UTF-8"; + }; + }; + + time.timeZone = "Europe/Oslo"; + +# programs.msmtp = { +# enable = false; +# accounts = { +# default = { +# auth = false; +# tls = false; +# tls_starttls = false; +# port = 24; +# from = "rossby@oceanbox.io"; +# host = "smtpgw.itpartner.no"; +# # user = "utvikling"; +# # password = "S0m3rp0m@de#21!"; +# }; +# }; +# defaults = { +# aliases = "/etc/aliases"; +# }; +# }; + + environment.etc = { + "aliases" = { + text = '' + root: jonas.juselius@oceanbox.io + ''; + mode = "0644"; + }; + }; + + features = { + os = { + # boot.uefi = true; + adminAuthorizedKeys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHVwcJOtx9YTWy+aD4xGbyPFLOdMN6NqY8wcfDtHczyT Stig Rune Jensen" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfgY468dPNpdXZCkD9jw1p2qA0+z56Wi/c1VYE+riki Stig Rune Jensen" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst" + ]; + docker.enable = false; + }; + cachix.enable = false; + monitoring.nodeExporter.enable = false; + hpc.mft.enable = false; # Mellanox MFT + }; + + networking = { + domain = mkDefault "cluster.local"; +<<<<<<<< HEAD:rossby/default.nix + defaultGateway = mkDefault "172.16.239.1"; +======== + defaultGateway = mkDefault "10.255.241.1"; +>>>>>>>> ekman:ekman/default.nix + nameservers = mkDefault [ "8.8.8.8" ]; + search = mkDefault []; + extraHosts = import ./hosts.nix; + firewall.extraCommands = '' + iptables -I INPUT -s 172.16.239.0/24 -j ACCEPT + iptables -I INPUT -s 172.16.240.0/24 -j ACCEPT + ''; + }; + + environment.variables = {}; + + # systemd.services."serial-getty@ttyS0".enable = true; + + # environment.etc."beegfs/connauthfile" = { + # source = ./connauthfile; + # mode = "0400"; + # uid = 0; + # gid = 0; + # }; + + services.certmgr.validMin = "120h"; + services.certmgr.renewInterval = "30m"; + + nix.settings = { + max-jobs = 32; + trusted-users = [ "@wheel" ]; + substituters = [ + ]; + }; + + system.activationScripts = { + kraken-permissions.text = '' + chmod 755 /work/kraken + ''; + }; + }; + + slurm = { + features.hpc.slurm = { + enable = true; + client = true; + mungeKey = ./munge.key; + mungeUid = mkDefault 994; # hack + # pkey = "0x7666"; +<<<<<<<< HEAD:rossby/default.nix + controlMachine = "rossby-manage"; + mailDomain = "oceanbox.io"; + nodeName = [ + "c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "rossby-login Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "rossby-manage Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" +======== + controlMachine = "ekman-manage"; + mailDomain = "oceanbox.io"; + nodeName = [ + "c0-[1-18] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" + "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "ekman-manage Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN" +>>>>>>>> ekman:ekman/default.nix + ]; + partitionName = [ + "batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP" + "rossby Nodes=rossby-login MaxTime=1:00:00 State=UP" + "short Nodes=c0-[1-8] MaxTime=INFINITE State=UP" + "long Nodes=c0-[3-8] MaxTime=INFINITE State=UP" + "stats Nodes=c0-[7-8] MaxTime=INFINITE State=UP" + ]; + }; + }; + + compute = { + system.activationScripts = { + mkWorkDir.text = "mkdir -p /work"; + }; + + cluster.slurm = true; + + features = { + hpc = { + enable = true; + beegfs = { + enable = false; + # beegfs = { + # work = { + # mgmtdHost = "ibbeegfs0"; + # connAuthFile = "/etc/beegfs/connauthfile"; + # client = { + # enable = false; + # mountPoint = "/work"; + # }; + # }; + # }; + }; + }; + }; + }; + + k8s-node = { + features = { + k8s = { + enable = true; + node.enable = true; + clusterName = "rossby"; + initca = ./ca; + cidr = "10.10.0.0/16"; + master = { +<<<<<<<< HEAD:rossby/default.nix + name = "rossby-manage"; + address = "172.16.239.221"; +======== + name = "ekman-manage"; + address = "10.255.241.99"; +>>>>>>>> ekman:ekman/default.nix + # extraSANs = [ + # "rossby-manage.oceanbox.io" + # ]; + }; + ingressNodes = [ + "rossby-manage.oceanbox.io" + "rossby.oceanbox.io" + ]; + fileserver = "rossby-login"; + charts = { + acme_email = "acme@oceanbox.io"; + # grafana_smtp_user = "utvikling"; + # grafana_smtp_password = "S0m3rp0m@de#21!"; + }; + }; + }; + + system.activationScripts = { + copyCaKey.text = "cp ${./ca}/ca-key.pem /var/lib/kubernetes/secrets"; + }; + + # services.kubernetes.kubelet.extraSANs = mkSANs { + # name = cfg.name; + # address = cfg.address; + # }; + }; + + shosts = { + environment.etc."ssh/shosts.equiv" = { + mode = "0644"; + uid = 0; + gid = 0; + text = '' + 172.16.239.210 + '' + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes; + }; + + programs.ssh.knownHosts = { +<<<<<<<< HEAD:rossby/default.nix + rossby-manage = { + hostNames = [ + "manage" "manage.compute.local" "rossby-manage.oceanbox.io" "172.16.239.221" "172.16.240.221" +======== + ekman-manage = { + hostNames = [ + "ekman-manage" "ekman-manage.cluster.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.99" +>>>>>>>> ekman:ekman/default.nix + ]; + publicKeyFile = ./manage/ssh_host_key.pub; + }; + rossby-login = { + hostNames = [ +<<<<<<<< HEAD:rossby/default.nix + "rossby" "rpssby-login" "rossby-login.compute.local" "rossby.compute.local" "rossby.oceanbox.io" "172.16.239.222" "172.16.240.222" + ]; + publicKeyFile = ./login/ssh_host_key.pub; +======== + "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100" + ]; + publicKeyFile = ./logon/ssh_host_key.pub; + }; + fs-work = { + hostNames = [ + "fs-work" "fs-work.cluster.local" "10.255.241.90" "10.255.243.90" + ]; + publicKeyFile = ./fs-work/ssh_host_key.pub; + }; + fs-backup = { + hostNames = [ + "fs-backup" "fs-backup.cluster.local" "10.255.241.80" "10.255.243.80" + ]; + publicKeyFile = ./fs-backup/ssh_host_key.pub; +>>>>>>>> ekman:ekman/default.nix + }; + # fs-work = { + # hostNames = [ + # "fs-work" "fs-work.compute.local" "172.16.239.90" "172.16.240.210" + # ]; + # publicKeyFile = ./fs-work/ssh_host_key.pub; + # }; + } // builtins.foldl' (a: x: + let n = toString x.idx; + in a // { + "${x.name}" = { + hostNames = [ + "${x.name}" +<<<<<<<< HEAD:rossby/default.nix + "${x.name}.compute.local" + "172.16.239.${n}" + "172.16.240.${n}" +======== + "${x.name}.cluster.local" + "10.255.241.${n}" + "10.255.243.${n}" +>>>>>>>> ekman:ekman/default.nix + ]; + publicKeyFile = x.pubkey; + }; + }) {} computeNodes; + + environment.systemPackages = [ + openssh-shosts + pkgs.inotify-tools + pkgs.ceph + pkgs.ceph-client + ]; + + security.wrappers = { + ssh-keysign = { + source = "${openssh-shosts}/libexec/ssh-keysign"; + owner = "root"; + group = "root"; + permissions = "u+rs,g+rx,o+rx"; + }; + }; + }; + + openssh-shosts = pkgs.openssh.overrideAttrs (attrs: { + buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ]; + doCheck = false; # the tests take hours + }); + +in { + options.cluster = { + compute = mkEnableOption "Enable compute node configs"; + }; + + options.cluster = { + k8sNode = mkEnableOption "Enable k8s node"; + }; + + options.cluster = { + slurm = mkEnableOption "Enable slurm"; + }; + + config = mkMerge [ + configuration + shosts + (mkIf config.cluster.compute compute) + (mkIf config.cluster.k8sNode k8s-node) + (mkIf config.cluster.slurm slurm) + ]; + + imports = [ + ../modules + ../nixos + ../users.nix + ]; +} + diff --git a/ekman/etcdCluster.nix b/ekman/etcdCluster.nix new file mode 100644 index 0000000..6396472 --- /dev/null +++ b/ekman/etcdCluster.nix @@ -0,0 +1,13 @@ +{ + enable = true; + existing = true; + nodes = + { + frontend = "10.255.241.99"; + fs-work = "10.255.241.90"; + fs-backup = "10.255.241.80"; + # ekman = "10.255.241.100"; + }; +} + + diff --git a/ekman/fs-backup/default.nix b/ekman/fs-backup/default.nix new file mode 100644 index 0000000..9e8c710 --- /dev/null +++ b/ekman/fs-backup/default.nix @@ -0,0 +1,180 @@ +{ pkgs ? import {} }: +let + name = "fs-backup"; + address = "10.255.241.80"; + etcdCluster = import ../etcdCluster.nix; +in { + fs-backup = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "fs" "fs-backup" ]; + deployment.targetHost = address; + system.autoUpgrade.enable = lib.mkForce false; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + xfsprogs + ]; + + boot.swraid = { + enable = true; + mdadmConf = '' + DEVICE partitions + ARRAY /dev/md/0 metadata=1.2 UUID=b743fdd4:5b339cc7:7c43f50f:3b81243e name=fs2:0 + ''; + }; + + systemd.services.restart-md0 = { + description = "restart /dev/md0"; + wantedBy = [ "multi-user.target" ]; + after = [ "sys-devices-virtual-block-md0.device" "-.mount" ]; + before = [ "backup.mount" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' + restart=0 + ${util-linux}/bin/lsblk -o MAJ:MIN -n /dev/md0 | grep -q "254:" || restart=1 + if [ $restart = 1 ]; then + ${mdadm}/bin/mdadm --stop /dev/md0 + ${mdadm}/bin/mdadm --assemble /dev/md0 + sleep 1 + fi + ''; + }; + + cluster = { + k8sNode = true; + slurm = false; + mounts = { + rdma.enable = false; + automount.enable = true; + home = false; + opt = false; + work = false; + data = false; + backup = false; + ceph = false; + }; + }; + + features.hpc.slurm.mungeUid = 996; + + features = { + host = { + inherit address; + inherit name; + }; + + os = { + externalInterface = "eno1"; + nfs.enable = true; + nfs.exports = '' + /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + k8s = { + enable = true; + node.enable = true; + master.enable = false; + inherit etcdCluster; + }; + }; + + systemd.services.rc-local = { + description = "rc.local script"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' + # if [ -e /sys/block/md126 ]; then + # echo "deadline" > /sys/block/md126/queue/scheduler + # # echo "4096" > /sys/block/md126/queue/nr_requests + # echo "4096" > /sys/block/md126/queue/read_ahead_kb + # echo "always" > /sys/kernel/mm/transparent_hugepage/enabled + # echo "always" > /sys/kernel/mm/transparent_hugepage/defrag + # fi + grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist + grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist + ''; + }; + + boot.kernel.sysctl = { + "vm.dirty_background_ratio" = 5; + "vm.dirty_ratio" = 10; + "vm.vfs_cache_pressure" = 50; + "vm.min_free_kbytes" = 262144; + }; + + networking = { + hostName = name; + interfaces.eno1 = { + useDHCP = false; + ipv4.addresses = [ { + address = address; + prefixLength = 24; + } ]; + ipv4.routes = [ + { + address = "10.255.244.0"; + prefixLength = 24; + via = "10.255.241.99"; + } + ]; + }; + interfaces.ibp59s0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.243.80"; + prefixLength = 24; + } ]; + }; + firewall = { + allowedTCPPorts = []; + allowedUDPPorts = []; + extraCommands = '' + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; + }; + + services.rpcbind.enable = true; + + fileSystems = { + "/exports/backup" = { + device = "/backup"; + options = [ "bind" ]; + }; + "/exports/ekman" = { + device = "/backup/ekman-nfs"; + options = [ "bind" ]; + }; + }; + + programs.singularity.enable = true; + + imports = [ + ./hardware-configuration.nix + ../cluster.nix + ../mounts.nix + ]; + }; + +} diff --git a/ekman/fs-backup/hardware-configuration.nix b/ekman/fs-backup/hardware-configuration.nix new file mode 100644 index 0000000..b0add6b --- /dev/null +++ b/ekman/fs-backup/hardware-configuration.nix @@ -0,0 +1,41 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "mpt3sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/19b7e607-b138-442a-9026-3ae1092046c9"; + fsType = "ext4"; + }; + + fileSystems."/backup" = + { device = "/dev/vg1/data"; + fsType = "xfs"; + options = [ "ro" "noauto" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.eno2.useDHCP = lib.mkDefault true; + # networking.interfaces.eno3.useDHCP = lib.mkDefault true; + # networking.interfaces.eno4.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/fs-backup/ssh_host_key.pub b/ekman/fs-backup/ssh_host_key.pub new file mode 100644 index 0000000..cf287ba --- /dev/null +++ b/ekman/fs-backup/ssh_host_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCb3/Z4YEFmeeaqQYzATkHgNaDWaT6MjiQsrIFYlZsvdXyE5QjGZMeRBMgJQyoyeCm9ig3sMbY8qiWLc/Kg+rFnEy+4xO7GYzCfqo1YsETiD0K+m/TbOsz9M2m+N8k+REWpKp9njTOYwsjWbrQ6FQPeQaKmig7P0PXJsSS/SAUr0Ns98QtBA9KtfRbDDCUlG4T/+kfu4Y970u0ABOF+zrE2mzfuaSn7n/jJAcAusIUQ9iiTrI6WHAok8lg9jC/+piW6ToEN7/ZX/RDggj5U5ZeN2IsjD1TZq57b/v7BAVx2CA38IYjuYizJFIlL8o7hEP22an1i44crsaR8pwJcfeMWzQxVIm9TPhSZZB9Ibq1R1syNb9vH/svmR8csWl0Uemk3gUGBJwzltmuXhLBg8mobypfBThAI/ZQKslvM9G9r4LoeLmxOOUTbCfXhi3ndIkEC2vGvo74h58BQj/DvMJ3a+V6gTha4837e2LK3Fm/8XWtU42jNbNlmblhLZby7uKWWb6orCnxscRs3cjP3BObSh6h9DO2aXMTnGYSGFdUKrRLFRmzFr+MPdhyORqp+GQgtHkzFG9W3/otQHdDCcD2OeFOxwEr3jOwUgtGIO1ZmJFDB9EIVHWyRk519foeZq/EHcSmIxgbwTgkpi/w8NQ1j++2nDxQOUeWo3U8xiQhONQ== root@fs2 diff --git a/ekman/fs-work/default.nix b/ekman/fs-work/default.nix new file mode 100644 index 0000000..42ec4ca --- /dev/null +++ b/ekman/fs-work/default.nix @@ -0,0 +1,171 @@ +{ pkgs ? import {} }: +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; + # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; + # }) {}; + # pkgs = import {}; + etcdCluster = import ../etcdCluster.nix; + name = "fs-work"; + address = "10.255.241.90"; +in { + fs-work = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "fs" "fs-work" ]; + deployment.targetHost = address; + system.autoUpgrade.enable = lib.mkForce false; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + ]; + + cluster = { + k8sNode = true; + slurm = false; + mounts = { + rdma.enable = false; + automount.enable = true; + users = false; + opt = false; + work = false; + data = false; + ceph = false; + }; + }; + + features.hpc.slurm.mungeUid = 994; + + features = { + host = { + inherit address; + inherit name; + }; + + os = { + externalInterface = "enp33s0f3np3"; + nfs.enable = true; + nfs.exports = '' + /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + k8s = { + enable = true; + node.enable = true; + master.enable = false; + inherit etcdCluster; + }; + }; + + systemd.services.rc-local = { + description = "rc.local script"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + path = [ "/run/current-system/sw/" ]; + serviceConfig = { + Type = "oneshot"; + }; + script = '' + # if [ -e /sys/block/md126 ]; then + # echo "deadline" > /sys/block/md126/queue/scheduler + # # echo "4096" > /sys/block/md126/queue/nr_requests + # echo "4096" > /sys/block/md126/queue/read_ahead_kb + # echo "always" > /sys/kernel/mm/transparent_hugepage/enabled + # echo "always" > /sys/kernel/mm/transparent_hugepage/defrag + # fi + grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist + grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist + ''; + }; + + boot.kernel.sysctl = { + "vm.dirty_background_ratio" = 5; + "vm.dirty_ratio" = 10; + "vm.vfs_cache_pressure" = 50; + "vm.min_free_kbytes" = 262144; + }; + + networking = { + hostName = name; + interfaces.enp65s0f0np0 = { + useDHCP = false; + ipv4.addresses = [ + { + address = address; + prefixLength = 24; + } + ]; + ipv4.routes = [ + { + address = "10.255.242.0"; + prefixLength = 24; + via = "10.255.241.100"; + } + ]; + }; + interfaces.enp1s0f1np1 = { + useDHCP = false; + ipv4.addresses = [ + { + address = "10.255.244.90"; + prefixLength = 24; + } + ]; + }; + # interfaces."ibp65s0.7666" = { + # useDHCP = false; + # }; + interfaces.ibp1s0f0 = { + useDHCP = false; + ipv4.addresses = [ + { + address = "10.255.243.90"; + prefixLength = 24; + } + ]; + }; + firewall = { + allowedTCPPorts = []; + allowedUDPPorts = []; + extraCommands = '' + # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; + }; + + services.rpcbind.enable = true; + + fileSystems = { + "/exports/work" = { + device = "/work"; + options = [ "bind" ]; + }; + "/exports/opt" = { + device = "/opt"; + options = [ "bind" ]; + }; + }; + + programs.singularity.enable = true; + + imports = [ + ./hardware-configuration.nix + ../default.nix + ../mounts.nix + ]; + }; +} diff --git a/ekman/fs-work/hardware-configuration.nix b/ekman/fs-work/hardware-configuration.nix new file mode 100644 index 0000000..1148099 --- /dev/null +++ b/ekman/fs-work/hardware-configuration.nix @@ -0,0 +1,47 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "smartpqi" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/21d2c2c7-4968-432d-a5c4-08116147b816"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/49B0-2597"; + fsType = "vfat"; + options = [ "fmask=0022" "dmask=0022" ]; + }; + + fileSystems."/work" = + { device = "/dev/mapper/data-work"; + fsType = "xfs"; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f0np0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0f1np1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp6s0f4u1u6.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/fs-work/ssh_host_key.pub b/ekman/fs-work/ssh_host_key.pub new file mode 100644 index 0000000..80a68fc --- /dev/null +++ b/ekman/fs-work/ssh_host_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxcJ+G4e6TEy0CZ66V6rRvODZ3QBgG5uIPElZr0lAFEe2FLPtiR57puXTV4tFhkY/TsVo0BE5/qDcBUzdRJ0ihuVVruvEUJDr+wDby1+V823xvPYT0IqPNvBBcR/FKIUTWFIhDHCntQ3MWvcp+bXX4ZH02QGmnSNNjyH6krOofuNz0/H8UxpLn/U40ruQx5SyT+uc7t0qryEUAwdvkQ6dGqyeCd4szSyQ1RfaTcd3qqXltxfKx5EbRAxD6pLoiKzYBGzWwKOLczNJGsLoMQ53KhEUyAdEOEx79qxoR4PU8P5q7osZD9NuloHgVT2P1hiNcsNmFzze3WW96PyK9L6Y1NE2rfcrhHIHG8tK31YXBlFQWr/VuPHpvqt21z0r2tPJwM5ui1zgZ1tsEWuKURItqr5Xsxk3D3woY5hT4rtDkV0TLe8/PUTGkGtw15UbQ/BnsFctWe5JnTtZ+90Tjxl4Ty/h0AYilVff6I3tfEln9MBnYig/cOUX0lmo2lnYpUedHYq5LXaga15OW0uonMd8Wl0nSGbqpJ0Gmu+gONJU9AbGchgxxmo0LNbdQ4jANq11LhcK2UZpyPKA7zV0pcgo1WSQyewfdFYIOJzGJPbDhOai/0OAXJYPJa/QJvp08ULpGrUUYXtjmVLw0DCYihm4af84Gr5Fu7hqNO1dplbJh8w== root@nixos diff --git a/ekman/hive.nix b/ekman/hive.nix new file mode 100644 index 0000000..c28696d --- /dev/null +++ b/ekman/hive.nix @@ -0,0 +1,25 @@ +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; + # }) {}; + pkgs = import {}; + + ekman-manage = { + deployment = { + tags = [ "manage" "ekman" ]; + allowLocalDeployment = true; + targetHost = null; + }; + imports = [ ./manage ]; + }; + + ekman-login = import ./login { inherit pkgs; }; + c0 = import ./c0 { inherit pkgs; }; + c0x = import ./c0x { inherit pkgs; }; + c1 = import ./c1 { inherit pkgs; }; + fs-work = import ./fs-work { inherit pkgs; }; + fs-backup = import ./fs-backup { inherit pkgs; }; +in + { inherit ekman-manage; } // ekman-login // c0 // c0x // c1 // fs-work // fs-backup diff --git a/ekman/hosts.nix b/ekman/hosts.nix new file mode 100644 index 0000000..f800d32 --- /dev/null +++ b/ekman/hosts.nix @@ -0,0 +1,83 @@ +'' + 10.255.242.2 ekman-gw ekman-gw.cluster.local ekman-gw.cluster.local + 10.255.242.3 front-gw front-gw.cluster.local front-gw.cluster.local + + 10.255.241.90 fs-work fs-work.cluster.local + 10.255.241.90 nfs1 nfs1.cluster.local + 10.255.241.90 fs1 fs1.cluster.local + 10.255.241.80 fs-backup fs-backup.cluster.local + 10.255.241.80 fs2 fs2.cluster.local + 10.255.241.100 ekman ekman.cluster.local ekman.cluster.local + 10.255.241.100 etcd0 etcd0.cluster.local + 10.255.241.80 etcd1 etcd1.cluster.local + 10.255.241.90 etcd2 etcd2.cluster.local + 10.255.241.99 ekman-manage frontend frontend.cluster.local frontend.cluster.local + 10.255.243.99 ibfrontend ibfrontend.cluster.local ibfrontend.cluster.local + + 10.255.241.101 c0-1 c0-1.cluster.local + 10.255.241.102 c0-2 c0-2.cluster.local + 10.255.241.103 c0-3 c0-3.cluster.local + 10.255.241.104 c0-4 c0-4.cluster.local + 10.255.241.105 c0-5 c0-5.cluster.local + 10.255.241.106 c0-6 c0-6.cluster.local + 10.255.241.107 c0-7 c0-7.cluster.local + 10.255.241.108 c0-8 c0-8.cluster.local + 10.255.241.109 c0-9 c0-9.cluster.local + 10.255.241.110 c0-10 c0-10.cluster.local + 10.255.241.111 c0-11 c0-11.cluster.local + 10.255.241.112 c0-12 c0-12.cluster.local + 10.255.241.113 c0-13 c0-13.cluster.local + 10.255.241.114 c0-14 c0-14.cluster.local + 10.255.241.115 c0-15 c0-15.cluster.local + 10.255.241.116 c0-16 c0-16.cluster.local + 10.255.241.117 c0-17 c0-17.cluster.local + 10.255.241.118 c0-18 c0-18.cluster.local + + 10.255.241.121 c1-1 c1-1.cluster.local + 10.255.241.122 c1-2 c1-2.cluster.local + 10.255.241.123 c1-3 c1-3.cluster.local + 10.255.241.124 c1-4 c1-4.cluster.local + 10.255.241.125 c1-5 c1-5.cluster.local + 10.255.241.126 c1-6 c1-6.cluster.local + 10.255.241.127 c1-7 c1-7.cluster.local + 10.255.241.128 c1-8 c1-8.cluster.local + + 10.255.243.90 ibfs-work ibfs-work.cluster.local + 10.255.243.90 ibnfs1 ibnfs1.cluster.local + 10.255.243.90 ibfs1 ibfs1.cluster.local + 10.255.243.80 ibfs-backup ibfs-backup.cluster.local + 10.255.243.80 ibfs2 ibfs2.cluster.local + 10.255.243.100 ibekman ibekman.cluster.local + 10.255.243.100 ibetcd0 ibetcd0.cluster.local + 10.255.243.80 ibetcd1 ibetcd1.cluster.local + 10.255.243.90 ibetcd2 ibetcd2.cluster.local + + 10.255.243.101 ib0-1 ib0-1.cluster.local + 10.255.243.102 ib0-2 ib0-2.cluster.local + 10.255.243.103 ib0-3 ib0-3.cluster.local + 10.255.243.104 ib0-4 ib0-4.cluster.local + 10.255.243.105 ib0-5 ib0-5.cluster.local + 10.255.243.106 ib0-6 ib0-6.cluster.local + 10.255.243.107 ib0-7 ib0-7.cluster.local + 10.255.243.108 ib0-8 ib0-8.cluster.local + 10.255.243.109 ib0-9 ib0-9.cluster.local + 10.255.243.110 ib0-10 ib0-10.cluster.local + 10.255.243.111 ib0-11 ib0-1.cluster.local + 10.255.243.112 ib0-12 ib0-12.cluster.local + 10.255.243.113 ib0-13 ib0-13.cluster.local + 10.255.243.114 ib0-14 ib0-14.cluster.local + 10.255.243.115 ib0-15 ib0-15.cluster.local + 10.255.243.116 ib0-16 ib0-16.cluster.local + 10.255.243.117 ib0-17 ib0-17.cluster.local + 10.255.243.118 ib0-18 ib0-18.cluster.local + 10.255.243.118 ib0-18 ib0-19.cluster.local + + 10.255.243.121 ib1-1 ib1-1.cluster.local + 10.255.243.122 ib1-2 ib1-2.cluster.local + 10.255.243.123 ib1-3 ib1-3.cluster.local + 10.255.243.124 ib1-4 ib1-4.cluster.local + 10.255.243.125 ib1-5 ib1-5.cluster.local + 10.255.243.126 ib1-6 ib1-6.cluster.local + 10.255.243.127 ib1-7 ib1-7.cluster.local + 10.255.243.128 ib1-8 ib1-8.cluster.local +'' diff --git a/ekman/login/default.nix b/ekman/login/default.nix new file mode 100644 index 0000000..94baae8 --- /dev/null +++ b/ekman/login/default.nix @@ -0,0 +1,341 @@ +{ pkgs ? import {} }: +let + name = "rossby"; + address = "172.16.239.222"; + etcdCluster = import ../etcdCluster.nix; +in +{ + rossby-login = { config, pkgs, ... }: with pkgs; { + deployment.tags = [ "rossby-login" "login" ]; + deployment.targetHost = address; + system.autoUpgrade.enable = lib.mkForce false; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + cluster = { + compute = true; + k8sNode = true; + mounts = { + rdma.enable = false; + automount.enable = true; + users = false; + opt = false; + work = false; + data = false; + ceph = false; + }; + }; + + features = { + host = { + inherit name; + inherit address; + }; + + myvnc.enable = false; + + os = { + externalInterface = "enp129s0f0"; + nfs.enable = true; + nfs.exports = '' + /exports 172.16.239.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + hpc = { + slurm.server = false; + slurm.slurmrestd = false; + manager = false; + login = true; + knem = false; + }; + + k8s = { + master.enable = false; + node.enable = true; + inherit etcdCluster; + }; + + desktop.enable = false; + # server.enable = true; + monitoring = { + # server = { + # enable = false; + # scrapeHosts = [ "rossby-manage" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes); + # defaultAlertReceiver = { + # email_configs = [ + # { to = "jonas.juselius@oceanbox.io"; } + # ]; + # }; + # pageAlertReceiver = { + # webhook_configs = [ + # { + # url = "https://prometheus-msteams.k2.itpartner.no/rossby"; + # http_config = { + # tls_config = { insecure_skip_verify = true; }; + # }; + # } + # ]; + # }; + # }; + # webUI.enable = false; + # webUI.acmeEmail = "innovasjon@itpartner.no"; + # webUI.allow = [ + # "10.1.2.0/24" + # "172.19.254.0/24" + # "172.19.255.0/24" + # ]; + infiniband-exporter = { + enable = true; + nameMap = '' + # 0xe8ebd3030024981e "c0-1" + ''; + }; + slurm-exporter = { + enable = true; + port = 6080; + }; + }; + }; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6; + services.flannel.iface = "enp129s0f0"; + + networking = { + useDHCP = false; + hostName = name; +<<<<<<<< HEAD:rossby/login/default.nix + interfaces.enp129s0f0 = { + useDHCP = false; + ipv4.addresses = [ { + inherit address; +======== + interfaces.enp33s0f3np3 = { + useDHCP = false; + ipv4.addresses = [ { + address = address; +>>>>>>>> ekman:ekman/login/default.nix + prefixLength = 24; + } ]; + # ipv4.routes = [ + # { + # address = "10.255.244.0"; + # prefixLength = 24; + # via = "10.255.241.99"; + # } + # ]; + }; + # interfaces."ibp65s0f0" = { + # useDHCP = false; + # ipv4.addresses = [ { + # address = "10.255.243.100"; + # prefixLength = 24; + # } ]; + # }; + # interfaces."enp65s0f1np1" = { + # useDHCP = false; + # ipv4.addresses = [ { + # address = "10.255.244.100"; + # prefixLength = 24; + # } ]; + # }; + # interfaces.enp33s0f0np0 = { + # useDHCP = false; + # ipv4.addresses = [ { + # address = "10.255.242.2"; + # prefixLength = 24; + # } ]; + # ipv4.routes = [ + # { + # address = "10.1.8.0"; + # prefixLength = 24; + # via = "10.255.242.1"; + # } + # { + # address = "10.1.30.0"; + # prefixLength = 24; + # via = "10.255.242.1"; + # } + # ]; + # }; + defaultGateway = "172.16.239.1"; + firewall = { + allowedTCPPorts = [ 4443 ]; + extraCommands = '' + # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + ''; + }; + }; + + fileSystems = { + "/exports/home" = { + device = "/home"; + options = [ "bind" ]; + }; + "/exports/opt/bin" = { + device = "/opt/bin"; + options = [ "bind" ]; + }; + "/exports/opt/sif" = { + device = "/opt/sif"; + options = [ "bind" ]; + }; + "/exports/opt/singularity" = { + device = "/opt/singularity"; + options = [ "bind" ]; + }; + "/exports/nfs-provisioner" = { + device = "/vol/nfs-provisioner"; + options = [ "bind" ]; + }; + "/users" = { + device = "/home"; + options = [ "bind" ]; + }; + "/vol/local-storage/vol1" = { + device = "/vol/vol1"; + options = [ "bind" ]; + }; + "/vol/local-storage/vol2" = { + device = "/vol/vol2"; + options = [ "bind" ]; + }; + }; + + nix.extraOptions = '' + # secret-key-files = /etc/nix/rossby.key + ''; + + # services.xserver = { + # enable = false; + # enableCtrlAltBackspace = true; + # layout = "us"; + # xkbVariant = "altgr-intl"; + # xkbOptions = "eurosign:e"; + # displayManager = { + # gdm.enable = false; + # job.logToFile = true; + # }; + # # desktopManager.xfce.enable = true; + # }; + + services.prometheus.alertmanager.configuration.global = { + smtp_smarthost = "smtpgw.itpartner.no"; + # smtp_auth_username = "utvikling"; + # smtp_auth_password = "S0m3rp0m@de#21!"; + smtp_hello = "rossby.oceanbox.io"; + smtp_from = "noreplyrossby.oceanbox.io"; + }; + + # services.nginx = { + # virtualHosts = { + # "ds.matnoc.regnekraft.io" = { + # forceSSL = true; + # enableACME = true; + # serverAliases = []; + # locations."/" = { + # proxyPass = "http://localhost:9088"; + # proxyWebsockets = false; + # extraConfig = '' + # allow 10.1.2.0/24; + # allow 172.19.254.0/24; + # allow 172.19.255.0/24; + # deny all; + # ''; + # }; + # }; + # }; + # }; + + # services.gitlab-runner = { + # enable = true; + # extraPackages = with pkgs; [ + # singularity + # ]; + # concurrent = 4; + # services = { + # sif = { + # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; + # executor = "shell"; + # tagList = [ "rossby" "sif" ]; + # }; + # }; + # }; + + # security.sudo.extraConfig = '' + # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity + # ''; + + security.pam = { + services.sshd.googleAuthenticator.enable = true; + loginLimits = [ + { + domain = "@users"; + item = "rss"; + type = "hard"; + value = 16000000; + } + { + domain = "@users"; + item = "cpu"; + type = "hard"; + value = 180; + } + ]; + }; + + system.activationScripts = { + home-permissions.text = '' + chmod 755 /home/olean + chmod 755 /home/frankgaa + chmod 755 /home/jonas + chmod 755 /home/mrtz + chmod 755 /home/avle + chmod 755 /home/stig + chmod 755 /home/bast + chmod 755 /home/simenlk + chmod 755 /work/kraken + ''; + }; + + + # ssh-rsa is deprecated, but putty/winscp users use it + services.openssh.extraConfig = '' + # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 + PubkeyAuthOptions verify-required + ''; + + environment.systemPackages = []; + + virtualisation.docker.enable = pkgs.lib.mkForce true; + + services.tailscale = { + enable = true; + authKeyFile = "/var/lib/secrets/tailscale.key"; + useRoutingFeatures = "server"; # for exit-node usage + extraUpFlags = [ + "--login-server=https://headscale.svc.oceanbox.io" + "--accept-dns" + "--advertise-exit-node" + "--advertise-tags=tag:rossby" + ]; + }; + + imports = [ + ./hardware-configuration.nix + ../default.nix + ../mounts.nix + ../myvnc.nix + ]; + }; +} + diff --git a/ekman/login/hardware-configuration.nix b/ekman/login/hardware-configuration.nix new file mode 100644 index 0000000..acc3e52 --- /dev/null +++ b/ekman/login/hardware-configuration.nix @@ -0,0 +1,42 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-amd" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/e859992d-1627-42cc-a59c-178d596fc220"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/5281-9FFF"; + fsType = "vfat"; + options = [ "fmask=0077" "dmask=0077" ]; + }; + + swapDevices = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp129s0f0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp129s0f1.useDHCP = lib.mkDefault true; + # networking.interfaces.enp129s0f2.useDHCP = lib.mkDefault true; + # networking.interfaces.enp129s0f3.useDHCP = lib.mkDefault true; + # networking.interfaces.enp65s0np0.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/ekman/login/ssh_host_key.pub b/ekman/login/ssh_host_key.pub new file mode 100644 index 0000000..8d66cd4 --- /dev/null +++ b/ekman/login/ssh_host_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSfhEB+DPHCWHmdzf+Xea7N9A/Pd98Nh9ChcQqVcjJNUMNeOta3NIdW5m9sDhWlB6kioMEJa4DbdAfJsKciGnSh9VJeerfnPFgov3WIr5RfoTAfKRE5GKW28c1f4m0WpfZkh5gvS0mAT7jkxwvTcQI9PViteIb4muAlbLPCqDUhFWC9yzSHzSt9MSMjpPY18GlsWyPM2ctT3OUq28wWlcleu7pFEFIpLKj+tqpBDBvS1I0fKagR0eDmCUGGbT8Fth+Jodt66k+QmJ28A58DfiDPECeN1/AzhwOMRYYMLSFmrLRmy/gML1egiZM+zwDKTRQ9lHhFjO8SOfq0F/A4NxFXegmXI1QbbtsG8KrLEDd9a8pZG99YdRa1dE5TA3H5iAnekN7rO0vfnlJfGMQOdI2I9JtqX48k98wUjxmv0EiUJV68j0rQaEwDQFjbEmiN2nEA9sm6+CMKCf3PywvDAijzAzSicA5Pm9MfNQLiXTXZbB7biJlNLfSlSOZrOP7Rd1VZ+bF9lYou88Y3/BZ/S23NdMVr9DEVYu2pUOCEgeXwVW32LHEgDfUslp7ZDUICGgLMQX5Wpz8f+zByRUmE2ovhCjcT71TVn/paIqjjy+2sgDX+BVSULQQAoPS56by1jfMIZjjNqDWE86yY3uWkeobmY5C55s8q2Hlyv+8meq6LQ== root@nixos diff --git a/ekman/manage/default.nix b/ekman/manage/default.nix new file mode 100644 index 0000000..b8065c5 --- /dev/null +++ b/ekman/manage/default.nix @@ -0,0 +1,354 @@ +<<<<<<< HEAD:configuration.nix +_: +{ + imports = [ ./rossby/manage ]; +======= +{ pkgs, ...}: +let + computeNodes = + import ../c0/nodes.nix ++ + import ../c1/nodes.nix ++ + [ + rec { + idx = 100; + name = "ekman"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ../login/ekman.pub; + } + rec { + idx = 90; + name = "fs-work"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ../fs-work/fs-work.pub; + } + rec { + idx = 81; + name = "fs-backup"; + address = "10.255.241.${toString idx}"; + ipoib = "10.255.243.${toString idx}"; + pubkey = ../fs-backup/fs-backup.pub; + } + ]; + etcdCluster = import ../etcdCluster.nix; + name = "ekman-manage"; + address = "10.255.241.99"; + ipoib = "10.255.243.99"; +in { + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + environment.systemPackages = with pkgs; [ + rdma-core + hwloc + headscale + ]; + + cluster = { + k8sNode = true; + compute = false; + slurm = true; + mounts = { + rdma.enable = true; + automount.enable = true; + home = true; + opt = true; + work = true; + data = true; + backup = true; + ceph = true; + }; + }; + + features = { + desktop.enable = false; + cachix.enable = false; + + host = { + inherit address; + inherit name; + }; + + myvnc.enable = false; + + os = { + externalInterface = "eno1"; + nfs.enable = false; + nfs.exports = '' + /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + hpc = { + slurm.server = true; + slurm.slurmrestd = false; + slurm.mungeUid = 996; + frontend = true; + }; + + k8s = { + master.enable = true; + node.enable = true; + nodes = computeNodes; + inherit etcdCluster; + }; + + monitoring = { + server = { + enable = false; + scrapeHosts = [ + "ekman-manage" + "ekman" + "fs-work" + "fs-backup" + ] ++ (builtins.map (x: x.name) computeNodes); + defaultAlertReceiver = { + email_configs = [ + { to = "jonas.juselius@oceanbox.io"; } + ]; + }; + pageAlertReceiver = { + webhook_configs = [ + { + url = "https://prometheus-msteams.k2.itpartner.no/ekman"; + http_config = { + tls_config = { insecure_skip_verify = true; }; + }; + } + ]; + }; + }; + webUI.enable = false; + webUI.acmeEmail = "innovasjon@itpartner.no"; + webUI.allow = [ + "10.1.2.0/24" + "172.19.254.0/24" + "172.19.255.0/24" + ]; + infiniband-exporter = { + enable = true; + nameMap = '' + 0xe8ebd3030024a2c6 "ekman" + 0x0c42a10300ddc4bc "ekman-manage" + 0xe8ebd3030024a2ae "fs-work" + 0x1c34da0300787798 "fs-backup" + 0xe8ebd3030024981e "c0-1" + 0xe8ebd3030024a21a "c0-2" + 0xe8ebd30300249a3a "c0-3" + 0xe8ebd30300248b9e "c0-4" + 0xe8ebd30300248b86 "c0-5" + 0xe8ebd3030024998a "c0-6" + 0xe8ebd30300248b8e "c0-7" + 0xe8ebd3030024999e "c0-8" + 0xe8ebd30300248fca "c0-9" + 0xe8ebd3030024a216 "c0-10" + 0xe8ebd30300248b96 "c0-11" + 0xe8ebd30300248b9a "c0-12" + 0xe8ebd303002495d2 "c0-13" + 0xe8ebd303002495e2 "c0-14" + 0xe8ebd30300248f42 "c0-15" + 0xe8ebd303002495e6 "c0-16" + 0x0c42a10300dbe7f4 "c1-1" + 0x0c42a10300dbe7d8 "c1-2" + 0x0c42a10300dbe800 "c1-3" + 0x0c42a10300dbec80 "c1-4" + 0x0c42a10300dbea50 "c1-5" + 0x0c42a10300dbeb2c "c1-6" + 0x0c42a10300dbe7fc "c1-7" + 0x0c42a10300dbe5a0 "c1-8" + ''; + }; + slurm-exporter = { + enable = true; + port = 6080; + }; + }; + }; + + programs.singularity.enable = true; + + # services.udev.extraRules = '' + # KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666" + # ''; + + services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0''; + + services.flannel.iface = "eno2"; + + networking = { + useDHCP = false; + hostName = name; + interfaces.eno1 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.255.242.3"; + prefixLength = 24; + } ]; + }; + interfaces.eno2 = { + useDHCP = false; + ipv4.addresses = [ + { + inherit address; + prefixLength = 24; + } + ]; + }; + interfaces.ens2f1np1 = { + useDHCP = false; + ipv4.addresses = [ + { + address = "10.255.244.99"; + prefixLength = 24; + } + ]; + }; + interfaces.ibs2f0 = { + useDHCP = false; + ipv4.addresses = [ + { + address = ipoib; + prefixLength = 24; + } + ]; + }; + defaultGateway = "10.255.242.1"; + firewall = { + allowedTCPPorts = [ 4443 4725 ]; + extraCommands = '' + # needed for nodeport access on k1 and k2 + # iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3 + iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE + # iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE + # iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE + # iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99 + # iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -j SNAT --to-source 10.255.242.3 + ''; + }; + }; + + fileSystems = { + "/exports/public" = { + device = "/srv/public"; + options = [ "bind" ]; + }; + }; + + nix.extraOptions = '' + # secret-key-files = /etc/nix/ekman.key + ''; + + services.prometheus.alertmanager.configuration.global = { + smtp_smarthost = "smtpgw.itpartner.no"; + # smtp_auth_username = "utvikling"; + # smtp_auth_password = "S0m3rp0m@de#21!"; + smtp_hello = "ekman.oceanbox.io"; + smtp_from = "noreply@ekman.oceanbox.io"; + }; + + security.pam = { + services.sshd.googleAuthenticator.enable = true; + loginLimits = [ + { + domain = "@users"; + item = "rss"; + type = "hard"; + value = 16000000; + } + { + domain = "@users"; + item = "cpu"; + type = "hard"; + value = 180; + } + ]; + }; + + system.activationScripts = { + home-permissions.text = '' + chmod 755 /home/olean + chmod 755 /home/frankgaa + chmod 755 /home/jonas + chmod 755 /home/stig + chmod 755 /home/bast + chmod 755 /home/mrtz + chmod 755 /home/avle + chmod 755 /home/simenlk + chmod 755 /home/ole + ''; + }; + + # Use nvd to get package diff before apply + system.activationScripts.system-diff = { + supportsDryActivation = true; # safe: only outputs to stdout + text = '' + export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH" + if [ -e /run/current-system ]; then + ${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true + fi + ''; + }; + + # ssh-rsa is deprecated, but putty/winscp users use it + services.openssh.extraConfig = '' + # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 + PubkeyAuthOptions verify-required + ''; + + # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1; + + virtualisation.docker.enable = pkgs.lib.mkForce true; + +# Configuration for the coordination server for a tailscale network run using headscale. +# +# We can set it up to provide several exit nodes through which traffic can be routed. +# +# Servers can join using this command: +# `tailscale up --login-server net.b0.itpartner.no --accept-dns=false --advertise-exit-node` +# +# with the following config: +# +# service.tailscale = { +# enable = true; +# useRoutingFeatures = "server"; # for exit-node usage +# }; +# +# Clients can join using this command: +# `tailscale up --login-server net.b0.itpartner.no --accept-dns=false` +# + # services.headscale = { + # enable = true; + # address = "0.0.0.0"; + # port = 4725; # hscl + # settings = import ./headscale/settings.nix; + # }; + + services.tailscale = { + enable = true; + authKeyFile = "/var/lib/secrets/tailscale.key"; + useRoutingFeatures = "both"; # for exit-node usage + extraUpFlags = [ + "--login-server=https://headscale.svc.oceanbox.io" + "--accept-dns=false" + "--advertise-exit-node" + ]; + }; + + imports = [ + ./hardware-configuration.nix + ../default.nix + ../mounts.nix + ../myvnc.nix + ]; +>>>>>>> ekman:ekman/manage/default.nix +} diff --git a/ekman/manage/hardware-configuration.nix b/ekman/manage/hardware-configuration.nix new file mode 100644 index 0000000..84acc26 --- /dev/null +++ b/ekman/manage/hardware-configuration.nix @@ -0,0 +1,27 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/099be829-726d-44c9-b113-4c2604a663c5"; + fsType = "ext4"; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/8111-9F24"; + fsType = "vfat"; + }; + + swapDevices = [ ]; +} diff --git a/ekman/manage/ssh_host_key.pub b/ekman/manage/ssh_host_key.pub new file mode 100644 index 0000000..b4c0d9c --- /dev/null +++ b/ekman/manage/ssh_host_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkSSuHkieXwgFMKRy4MjwjNrJEItWbQHeAAH+Zn1YZp root@localhost diff --git a/ekman/mounts.nix b/ekman/mounts.nix new file mode 100644 index 0000000..73bfacc --- /dev/null +++ b/ekman/mounts.nix @@ -0,0 +1,140 @@ +{ lib, config, ... }: +with lib; +let + cfg = config.cluster.mounts; + + options = + [ "soft" "defaults" "vers=4.2" ] ++ + (if cfg.automount.enable then [ "noauto" "x-systemd.automount" ] else []); + + users = + if cfg.users then { +<<<<<<<< HEAD:rossby/mounts.nix + "/users" = { + device = "172.16.239.222:/home"; +======== + "/frontend" = { + device = "10.255.241.100:/home"; +>>>>>>>> ekman:ekman/mounts.nix + fsType = "nfs4"; + options = [ + "soft" + "defaults" + "noauto" + "x-systemd.automount" + ]; + }; + } else {}; + + opt = + let + server = "172.16.239.222"; + in + if cfg.opt then { + "/opt/bin" = { + device = "${server}:/opt/bin"; + fsType = "nfs4"; + inherit options; + }; + "/opt/sif" = { + device = "${server}:/opt/sif"; + fsType = "nfs4"; + inherit options; + }; + "/opt/singularity" = { + device = "${server}:/opt/singularity"; + fsType = "nfs4"; + inherit options; + }; + } else {}; + + data = + if cfg.ceph then { + "/data" = { + device = "/ceph"; + options = [ "bind" ]; + }; + } else {}; + + work = + if cfg.work then { + "/work" = { + device = "172.16.240.210:/work"; + fsType = "nfs4"; + options = options ++ (if cfg.rdma.enable then [ "rdma" ] else []); + }; + } else {}; + + ceph = + if cfg.ceph then { + "/ceph" = { + device = "oceanbox@.data=/"; + fsType = "ceph"; + options = [ + "mon_addr=172.16.239.211/172.16.239.212/172.16.239.213:6789" + "_netdev" + ]; + }; + } else {}; + +<<<<<<<< HEAD:rossby/mounts.nix + fileSystems = users // opt // data // work // ceph; +======== + fileSystems = users // opt // data // work // backup // ceph; +>>>>>>>> ekman:ekman/mounts.nix + + automount = mountpoint: + if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then + [{ + wantedBy = [ "multi-user.target" ]; + automountConfig = { + TimeoutIdleSec = "600"; + }; + where = mountpoint; + }] + else []; + + automounts = + [] ++ + automount "/work" ++ + automount "/opt" ++ + automount "/backup" ++ + automount "/data"; + + cephConf = + if cfg.ceph then { + "ceph/ceph.conf" = { + text = '' + [global] + mon_host = 172.16.239.211:6789,172.16.239.212:6789,172.16.239.213:6789 + log file = /tmp/ceph-$pid.log + [client.oceanbox] + key = replaceme + [client.rbd] + key = replaceme + ''; + mode = "0660"; + group = "admin"; + }; + } else {}; + +in +{ + options.cluster.mounts = { + rdma.enable = mkEnableOption "Enable NFS over RDMA"; + gbe100.enable = mkEnableOption "Enable NFS over 100 GbE"; + automount.enable = mkEnableOption "Enable NFS automounting"; + users = mkEnableOption "Enable /users"; + opt = mkEnableOption "Enable /opt"; + data = mkEnableOption "Enable /data"; + work = mkEnableOption "Enable /work"; + ceph = mkEnableOption "Enable /ceph"; + }; + + config = { + fileSystems = fileSystems; + environment.etc = cephConf; + systemd.automounts = automounts; + boot.kernelModules = [ "rbd" ]; + }; +} diff --git a/ekman/munge.key b/ekman/munge.key new file mode 100644 index 0000000..0aeca32 --- /dev/null +++ b/ekman/munge.key @@ -0,0 +1,2 @@ +çŁ/iką/¨÷|ńRŻEĽRŽ$ĂQfj5ˇrdĐĄś7“{˘–99âTÂîۛĂi‹ÄŒ‰–,ЌÍhçďŮ8töv:%‘T” +|ČÚČ´ţΕ§VŒ00w|ŸĎŽ÷íŕ|Č_ŸY{3L_!F1TdÔ&F7ő™B°R \ No newline at end of file diff --git a/ekman/myvnc.nix b/ekman/myvnc.nix new file mode 100644 index 0000000..41846be --- /dev/null +++ b/ekman/myvnc.nix @@ -0,0 +1,44 @@ +{ pkgs, lib, config, ... }: +with lib; +let + cfg = config.features.myvnc; + + myvnc = pkgs.writeScriptBin "myvnc" '' + #!${pkgs.runtimeShell} + + uid=`id -u` + port=$((9000+$uid)) + shell=`getent passwd $(id -un) | awk -F : '{print $NF}'` + vnc=${pkgs.turbovnc}/bin/vncserver + # vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver + + case $1 in + -p|--port) shift; port=$1 ;; + kill|stop) + display=$($vnc -list | sed -n 's/^\(:[0-9]\+\).*/\1/p'| head -1) + $vnc -kill $display + exit 0 + ;; + esac + ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1 + [ $? = 1 ] && $vnc -rfbport $port + echo "Xvnc server is running on port $port." + # exec $shell -i + ''; + + configuration = { + services.xserver.windowManager.fluxbox.enable = true; + environment.systemPackages = with pkgs; [ + alacritty + myvnc + ]; + }; +in { + options.features.myvnc = { + enable = mkEnableOption "Enable mynvc script"; + }; + + config = mkMerge [ + (mkIf config.features.myvnc.enable configuration) + ]; +} diff --git a/ekman/overlays.nix b/ekman/overlays.nix new file mode 100644 index 0000000..87e6e6f --- /dev/null +++ b/ekman/overlays.nix @@ -0,0 +1,31 @@ +self: super: +let + msmtp = super.msmtp.overrideAttrs (attrs: rec { + configureFlags = attrs.configureFlags ++ [ "--with-tls=openssl" ]; + buildInputs = attrs.buildInputs ++ [ super.openssl ]; + }); + + lib = super.lib; + squashfsTools = super.squashfsTools; + cryptsetup = super.cryptsetup; + + singularity = super.singularity.overrideAttrs (attrs: rec { + installPhase = '' + runHook preInstall + make -C builddir install LOCALSTATEDIR=$out/var + chmod 755 $out/libexec/singularity/bin/starter-suid + + # Explicitly configure paths in the config file + sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf + sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf + sed -i 's|^shared loop devices = no|shared loop devices = yes|' $out/etc/singularity/singularity.conf + + runHook postInstall + ''; + }); +in +{ + inherit singularity; + # inherit msmtp; +} + diff --git a/rossby/c0/default.nix b/rossby/c0/default.nix index 757a23d..e5e0519 100644 --- a/rossby/c0/default.nix +++ b/rossby/c0/default.nix @@ -35,7 +35,11 @@ let mkCompute = host: let +<<<<<<<< HEAD:rossby/c0/default.nix hw = ./hardware-config.d + "/${host.name}.nix"; +======== + hw = ./hardware-configuration.d + "/${host.name}.nix"; +>>>>>>>> ekman:ekman/c0/default.nix in { "${host.name}" = { cluster = { diff --git a/rossby/default.nix b/rossby/default.nix index 0cf67cc..a322082 100644 --- a/rossby/default.nix +++ b/rossby/default.nix @@ -99,7 +99,11 @@ let networking = { domain = mkDefault "cluster.local"; +<<<<<<<< HEAD:rossby/default.nix defaultGateway = mkDefault "172.16.239.1"; +======== + defaultGateway = mkDefault "10.255.241.1"; +>>>>>>>> ekman:ekman/default.nix nameservers = mkDefault [ "8.8.8.8" ]; search = mkDefault []; extraHosts = import ./hosts.nix; @@ -144,12 +148,22 @@ let mungeKey = ./munge.key; mungeUid = mkDefault 994; # hack # pkey = "0x7666"; +<<<<<<<< HEAD:rossby/default.nix controlMachine = "rossby-manage"; mailDomain = "oceanbox.io"; nodeName = [ "c0-[1-16] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" "rossby-login Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" "rossby-manage Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" +======== + controlMachine = "ekman-manage"; + mailDomain = "oceanbox.io"; + nodeName = [ + "c0-[1-18] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" + "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=2 RealMemory=256000 TmpDisk=500000 State=UNKNOWN" + "ekman-manage Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN" +>>>>>>>> ekman:ekman/default.nix ]; partitionName = [ "batch Nodes=c0-[1-16] Default=YES MaxTime=INFINITE State=UP" @@ -197,8 +211,13 @@ let initca = ./ca; cidr = "10.10.0.0/16"; master = { +<<<<<<<< HEAD:rossby/default.nix name = "rossby-manage"; address = "172.16.239.221"; +======== + name = "ekman-manage"; + address = "10.255.241.99"; +>>>>>>>> ekman:ekman/default.nix # extraSANs = [ # "rossby-manage.oceanbox.io" # ]; @@ -237,17 +256,41 @@ let }; programs.ssh.knownHosts = { +<<<<<<<< HEAD:rossby/default.nix rossby-manage = { hostNames = [ "manage" "manage.compute.local" "rossby-manage.oceanbox.io" "172.16.239.221" "172.16.240.221" +======== + ekman-manage = { + hostNames = [ + "ekman-manage" "ekman-manage.cluster.local" "frontend.oceanbox.io" "10.255.241.99" "10.255.243.99" +>>>>>>>> ekman:ekman/default.nix ]; publicKeyFile = ./manage/ssh_host_key.pub; }; rossby-login = { hostNames = [ +<<<<<<<< HEAD:rossby/default.nix "rossby" "rpssby-login" "rossby-login.compute.local" "rossby.compute.local" "rossby.oceanbox.io" "172.16.239.222" "172.16.240.222" ]; publicKeyFile = ./login/ssh_host_key.pub; +======== + "ekman" "ekman.cluster.local" "ekman.oceanbox.io" "10.255.241.100" "10.255.243.100" + ]; + publicKeyFile = ./logon/ssh_host_key.pub; + }; + fs-work = { + hostNames = [ + "fs-work" "fs-work.cluster.local" "10.255.241.90" "10.255.243.90" + ]; + publicKeyFile = ./fs-work/ssh_host_key.pub; + }; + fs-backup = { + hostNames = [ + "fs-backup" "fs-backup.cluster.local" "10.255.241.80" "10.255.243.80" + ]; + publicKeyFile = ./fs-backup/ssh_host_key.pub; +>>>>>>>> ekman:ekman/default.nix }; # fs-work = { # hostNames = [ @@ -261,9 +304,15 @@ let "${x.name}" = { hostNames = [ "${x.name}" +<<<<<<<< HEAD:rossby/default.nix "${x.name}.compute.local" "172.16.239.${n}" "172.16.240.${n}" +======== + "${x.name}.cluster.local" + "10.255.241.${n}" + "10.255.243.${n}" +>>>>>>>> ekman:ekman/default.nix ]; publicKeyFile = x.pubkey; }; diff --git a/hive.nix b/rossby/hive.nix similarity index 74% rename from hive.nix rename to rossby/hive.nix index b504449..b74abfe 100644 --- a/hive.nix +++ b/rossby/hive.nix @@ -12,11 +12,11 @@ let allowLocalDeployment = true; targetHost = null; }; - imports = [ ./rossby/manage ]; + imports = [ ./manage ]; }; - rossby-login = import ./rossby/login { inherit pkgs; }; - c0 = import ./rossby/c0 { inherit pkgs; }; - fs-work = import ./rossby/fs-work { inherit pkgs; }; + rossby-login = import ./login { inherit pkgs; }; + c0 = import ./c0 { inherit pkgs; }; + fs-work = import ./fs-work { inherit pkgs; }; in { inherit rossby-manage; } // rossby-login // c0 // fs-work diff --git a/rossby/login/default.nix b/rossby/login/default.nix index eef6fa7..94baae8 100644 --- a/rossby/login/default.nix +++ b/rossby/login/default.nix @@ -113,10 +113,17 @@ in networking = { useDHCP = false; hostName = name; +<<<<<<<< HEAD:rossby/login/default.nix interfaces.enp129s0f0 = { useDHCP = false; ipv4.addresses = [ { inherit address; +======== + interfaces.enp33s0f3np3 = { + useDHCP = false; + ipv4.addresses = [ { + address = address; +>>>>>>>> ekman:ekman/login/default.nix prefixLength = 24; } ]; # ipv4.routes = [ diff --git a/rossby/mounts.nix b/rossby/mounts.nix index 18520eb..73bfacc 100644 --- a/rossby/mounts.nix +++ b/rossby/mounts.nix @@ -9,8 +9,13 @@ let users = if cfg.users then { +<<<<<<<< HEAD:rossby/mounts.nix "/users" = { device = "172.16.239.222:/home"; +======== + "/frontend" = { + device = "10.255.241.100:/home"; +>>>>>>>> ekman:ekman/mounts.nix fsType = "nfs4"; options = [ "soft" @@ -72,7 +77,11 @@ let }; } else {}; +<<<<<<<< HEAD:rossby/mounts.nix fileSystems = users // opt // data // work // ceph; +======== + fileSystems = users // opt // data // work // backup // ceph; +>>>>>>>> ekman:ekman/mounts.nix automount = mountpoint: if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then