fix(gitea-runner): Use public ingress

Also switches to podman

.envrc

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+watch_file nix/sources.json
+
+# Load .env file if it exists
+dotenv_if_exists
+
+# Set npins dir
+export NPINS_DIRECTORY="nix"
+
+# Activate development shell
+use nix

.gitignore

@@ -1,6 +1,10 @@
 *.pem
 *.csr
+*.key
 result
 result-*
 gcroots/
 ca
+configuration.nix
+system
+.direnv

bin/reuid-slurm.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+pwunconv
+sed -i 's/slurm:!:[0-9]\+:[0-9]\+:/slurm:!:401:401:/;
+        s/munge:!:[0-9]\+:[0-9]\+:/munge:!:402:402:/' /etc/passwd
+pwconv
+
+grpunconv
+sed -i 's/slurm:x:[0-9]\+:/slurm:x:401:/;
+        s/munge:x:[0-9]\+:/munge:x:402:/' /etc/group
+grpconv
+
+systemctl stop munged
+rm -rf /run/munge
+chown -R munge:munge /etc/munge /var/lib/munge
+systemctl start munged
+
+systemctl stop slurmd
+rm -rf /run/slurm
+chown -R slurm:slurm /etc/slurm
+systemctl start slurmd
+
+if [ -d /var/spool/slurmctld ]; then
+    systemctl stop slurmctld
+    rm -rf /run/slurm /run/slurmctld
+    systemctl start slurmctld
+fi
+

bin/slurm-gen-jwt_hs256.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+dd if=/dev/random of=jwt_hs256.key bs=32 count=1
+chmod 400 jwt_hs256.key

bin/slurm-gen-mungekey.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+mungekey -c -b 2048 -k munge.key

configuration.nix

@@ -1,4 +0,0 @@
-_:
-{
-  imports = [ ./rossby/manage ];
-}

dns.nix

@@ -0,0 +1,27 @@
+{ pkgs, ... }:
+let
+  zonefile = builtins.toFile "obx.zone" (builtins.readFile ./obx.zone);
+in {
+  services.coredns = {
+    enable = true;
+    config = ''
+      . {
+          errors
+          log . {
+            class error
+          }
+          file ${zonefile} obx.
+          cache 30 {
+             disable success cluster.local
+             disable denial cluster.local
+          }
+          header {
+            response set ra
+          }
+          forward . 8.8.8.8 8.8.4.4
+          loop
+          reload
+      }
+    '';
+  };
+}

ekman/bin/adduser.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+id=$1
+user=$2
+name="$3"
+
+grp="\    $user = { gid = "$id"; };"
+
+read -d '' usr << EOF
+\\\   $user = {\\\n\
+      description = "$name";\\\n\
+      home = "/home/$user";\\\n\
+      group = "$user";\\\n\
+      extraGroups = [\\\n\
+        "users"\\\n\
+        "docker"\\\n\
+      ];\\\n\
+      uid = $id;\\\n\
+      isNormalUser = true;\\\n\
+      createHome = true;\\\n\
+      openssh.authorizedKeys.keys = [];\\\n\
+    };\\\n\
+
+EOF
+
+sed -i "
+/# @grp@/i $grp
+/# @usr@/i $usr
+" stokes/users.nix

ekman/bin/initca.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.."
+
+if [ "x$1" = "x" ]; then
+    echo "usage: initca.sh {cluster}"
+    exit 1
+fi
+
+ca=$TOP/modules/initca.nix
+
+cd $TOP/$1
+
+echo "--- Preparing CA certificate"
+nix-build -o ca $ca
+
+echo "--- Safeguarding CA certificate"
+nix-store --add-root $(pwd)/ca --indirect -r $(nix-instantiate --add-root $ca)

ekman/c0/connauthfile

@@ -0,0 +1 @@
+莵q丘

ekman/c0/default.nix

@@ -0,0 +1,136 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
+  # }) {};
+  # pkgs = import <nixpkgs> {};
+
+  nodes = import ./nodes.nix;
+  has100GbE = [ "c0-18" ];
+
+  mkCompute =
+    host:
+    let
+      hw = ./hardware-configuration.d + "/${host.name}.nix";
+    in
+    {
+      "${host.name}" = {
+        deployment.tags = [
+          "compute"
+          "c0"
+          "cluster"
+        ];
+        deployment.targetHost = host.address;
+
+        cluster = {
+          compute = true;
+          k8sNode = true;
+          mounts = {
+            rdma.enable = true;
+            automount.enable = true;
+            users = true;
+            opt = true;
+            work = true;
+            data = false;
+            ceph = false;
+            backup = false;
+          }
+          // (
+            if (builtins.elem host.name has100GbE) then
+              {
+                data = true;
+                ceph = true;
+              }
+            else
+              { }
+          );
+        };
+
+        features = {
+          host = {
+            name = host.name;
+            address = host.address;
+          };
+          os.networkmanager.enable = false;
+          os.externalInterface = host.iface;
+          hpc.computeNode = true;
+          hpc.knem = true;
+        };
+
+        # services.udev.extraRules = ''
+        #   KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+        # '';
+
+        networking = {
+          useNetworkd = true;
+          hostName = host.name;
+          useDHCP = false;
+        };
+
+        # systemd.services.systemd-networkd-wait-online.enable =  false;
+
+        systemd.network = {
+          wait-online.ignoredInterfaces = [ "ibp1s0" ];
+          # wait-online.enable = false;
+          networks = {
+            "40-${host.iface}" = {
+              matchConfig.Name = host.iface;
+              address = [ "${host.address}/24" ];
+              # networkConfig = {
+              #   DNSDefaultRoute = true;
+              # };
+              routes = [
+                { Gateway = "10.255.241.1"; }
+                {
+                  Destination = "172.16.239.0/24";
+                  Gateway = "10.255.241.210";
+                }
+                {
+                  Destination = "10.255.242.0/24";
+                  Gateway = "10.255.241.100";
+                }
+              ];
+            };
+            "45-ibp1s0" = {
+              matchConfig.Name = "ibp1s0";
+              address = [ "${host.ipoib}/24" ];
+            };
+          }
+          // (
+            if (builtins.elem host.name has100GbE) then
+              {
+                "42-enp65s0f1np1" = {
+                  DHCP = "no";
+                  matchConfig.Name = "enp65s0f1np1 ";
+                  address = [ "${host.gbe100}/24" ];
+                };
+              }
+            else
+              { }
+          );
+        };
+
+        boot.kernelParams = [
+          "console=tty0"
+          "console=ttyS0,115200"
+        ];
+        systemd.services."serial-getty@ttyS0" = {
+          enable = true;
+          wantedBy = [ "getty.target" ];
+          serviceConfig.Restart = "always";
+        };
+
+        imports = [
+          hw
+          ../default.nix
+          ../mounts.nix
+          ./kernel.nix
+        ];
+      };
+    };
+in
+builtins.foldl' (a: n: a // mkCompute n) { } nodes

ekman/c0/hardware-configuration.d/c0-1.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/7e98da6d-a33d-4845-9123-7e351ad0a39e";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/6ACD-6E5F";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-10.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/26d04544-c68a-4e7a-a728-5676f472ebb7";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/7E82-333A";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-11.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/4a8e1c95-8bbb-40ed-8865-74bab73e4ee8";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/7D28-3770";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-12.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/cc8f9ec9-fe2c-4603-8ba1-2df9c8389cbf";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/B26B-7B6E";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-13.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/a3fd9989-866c-4d24-9c59-7c2803a28eae";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/8B58-FF6B";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-14.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/f4579aea-21f4-4d4f-9c65-53f9b540cac4";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/A20A-720E";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-15.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/4db39a06-d119-4efb-9111-41c85e622467";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/B8A4-E5C5";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-16.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/6923deaa-ab2e-45c1-bfe6-7fe8be6f1770";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/C9D4-6AE3";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-17.nix

@@ -0,0 +1,55 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/46dfa481-ccd4-4e4f-b1d7-6875d582f7cd";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/F71C-0BD2";
+      fsType = "vfat";
+    };
+
+#  fileSystems."/data" =
+#    { device = "/dev/disk/by-uuid/125c5d04-5266-43e8-887d-740d5944cb2b";
+#      fsType = "xfs";
+#      options = [
+#        "noatime"
+#        "nodiratime"
+#        "logbufs=7"
+#        "logbsize=255k"
+#        "largeio"
+#        "inode63"
+#        "swalloc"
+#        "allocsize=131071k"
+#      ];
+#    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.docker0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-18.nix

@@ -0,0 +1,44 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "sd_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/19d00648-5fbd-4464-93a0-e013d7f79d3a";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/68AF-2717";
+      fsType = "vfat";
+    };
+
+  fileSystems."/tmp" =
+    { device = "/dev/disk/by-uuid/bf1bede5-bc60-4603-874c-88ed85e6ab5f";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-2.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/d7fdc1a4-39fe-4260-be6c-1857ce972d7e";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/0B80-A90E";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-3.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/aa46e116-30eb-4e49-996b-7ee49fb1914c";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/8DF7-E9B8";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-4.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/886d3243-07c5-4679-9979-cef11a9b5cb1";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/6FA3-B1DE";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-5.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/db0635c5-e0bf-4d85-9834-f0374f8ffa9c";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/A3AE-176D";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-6.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/de9abab9-3280-4ef2-b467-ea09842b1be6";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/63E2-0C5A";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-7.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/22b9bf44-cafb-4f06-a23f-9b6f4a748e81";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/5CBE-8643";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-8.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/f972da1f-a7da-4c94-8bee-cf733a3e4aef";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/7031-4731";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/c0-9.nix

@@ -0,0 +1,37 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/30add79f-20c1-4030-8d72-47a3dc6b533d";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/716E-5AC4";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/c0/hardware-configuration.d/fix.sh

@@ -0,0 +1,12 @@
+mv c0-8.nix  c0-6.nix.tmp  
+mv c0-6.nix  c0-7.nix.tmp  
+mv c0-7.nix  c0-8.nix.tmp  
+mv c0-15.nix c0-10.nix.tmp 
+mv c0-14.nix c0-12.nix.tmp 
+mv c0-12.nix c0-14.nix.tmp 
+mv c0-10.nix c0-15.nix.tmp 
+
+for i in *.tmp; do
+  mv $i $(basename $i .tmp)
+done
+  

ekman/c0/kernel.nix

@@ -0,0 +1,58 @@
+{ pkgs, lib, ... }:
+let
+  kernel = pkgs.linuxPackages.kernel;
+  i40e = pkgs.stdenv.mkDerivation rec {
+    name = "i40e-${version}-${kernel.version}";
+    version = "2.13.10";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "dmarion";
+      repo = "i40e";
+      rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
+      sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
+    };
+
+    hardeningDisable = [ "pic" ];
+
+    configurePhase = ''
+      cd src
+      kernel_version=${kernel.modDirVersion}
+      sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
+      sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
+      export makeFlags="BUILD_KERNEL=$kernel_version"
+    '';
+
+    installPhase = ''
+      install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
+    '';
+
+    dontStrip = true;
+
+    enableParallelBuilding = true;
+
+    meta = {
+      description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
+      homepage = "https://github.com/dmarion/i40e";
+      license = lib.licenses.gpl2;
+    };
+  };
+in
+{
+  # i40e2 = i40e;
+  boot.kernelPackages = pkgs.linuxPackagesFor (
+    pkgs.linux_5_10.override {
+      argsOverride = rec {
+        src = pkgs.fetchurl {
+          url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
+          sha256 = "1nzhl1y6avfl77fyqwjwy3qc6679gp92k0d3aarscrdydcml5yid";
+        };
+        version = "5.10.239";
+        modDirVersion = "5.10.239";
+      };
+    }
+  );
+  # boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
+  # overlay = self: super: {
+  # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
+  # };
+}

ekman/c0/nodes.nix

@@ -0,0 +1,13 @@
+with builtins;
+let
+  nodes = genList (n: n + 1) 18; in
+map (n: (
+  rec {
+    idx = 100 + n;
+    iface = if n > 16 then "enp33s0f3np3" else "enp33s0f0np0";
+    name = "c0-${toString n}";
+    address = "10.255.241.${toString idx}";
+    ipoib = "10.255.243.${toString idx}";
+    gbe100 = "10.255.244.${toString idx}";
+    pubkey = ./. +  "/ssh_host_key.d/c0-${toString n}.pub";
+  })) nodes

ekman/c0/ssh_host_key.d/c0-1.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKMILMKLqs/kq0ToCFJRRIaJFPh0FP0IHGjrg1G+UXB5aepVQ0GgiFeG9RiAtXbCNEqJDTXfhRRY2FJoNOghFEZ0Z6ECfbvS79OT8XDu2iouMHmu3Xtbh0LGNdoV3LHoSOrcUZnQwQXeQu5J12AIbyC5ng3vOf3Ud9pVehM7WqRegSiaqacneISfq1JtHzWcO7P/Rd9DtyEuFDuTZHLSPrF4+bnbLW7WIgPGTzs28PvWurhmZg01ahcadACAeDmxVFHvSjZbNJbFMPVrNkJbNzXyw8v2WN8wISmiMbJ+ANi/zBgiyc+TOgnbb3us5m6+qkhK7MboQz22y3aFFb/hUENk5aQEpyp7zG5Ua80eGue3EZ5iLselHI7J6w3UUST0tzY7J5uwhKI6Vx6m70s5qUZWG0oF06Y/+xC6GNu5+JBDd7E7l/TjXTtSmlVXcUOjhqK+qtwOsNtFKdWGthdEKuNoa4sfv7tkmtXIvjBuceUMgaHB0hQYm34cj5k82GXdg8U5xkf7dQkv0M2VrPJ2f8KvjSlVM3YvMQ5Pb6WaB5kP0PGEsDY0X5OHMwc6UhYhngJo1sIlwYQwXVQtjj0wVO4jEuGZczgq3FqusHXbD8uOy3eQxLIZOebvzhAnzaDfXhyIM8jhG9WHQMprJf+No+LlisPEBOlnHcxylsCV8VGQ== root@nixos

ekman/c0/ssh_host_key.d/c0-10.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+VIsgEyusSjMuU26gQAYNoWs9iC1ErHVjmZNpbtzZ5IlR4RdsmsStf1QZUSG5FJRA2pMOaZD0QUOFI4t4k4VWsYwVXhgkcjF1u0+nlc8XMSihmIHsNPZNYBVRyuL4shegjezKn4EZoHn41UxAcpWAg9XDIRsENAm1G8RcPqwmhrLg4KpC482iuqJT+GavdEcV5rT0O3wCeq0EyFpyd8wX3JARKiyBA5HjyiVzJTeycda3ffudwDmsym4JQeYHretj5REc1Wn4qOVrn6m/L5K7ttysTXvF3aQOegxkOfFhFvqPYJm/I0J5e2U4yRbwaaTyG2acfdIFLu/wwJUEjN7iQ9pj+uCRqfPEtZj1xe6CxXYBQk932gq2H+Y9QsC898Ubq1G9js+AaT3vSIUenGzhGYZl4VIVWjN6h5JvyxAUAbhBSD3zvV1bx/vv1k4e9l7vk3HyZncEZyL81gIB49VKUkWtef9Z/+5778xFjYrz5ykqdkCZEPa6IpphQz9P59MB64rqfK6VNLHN0LwiGxaLP+ctnx6FLXxVYgUtGQS4bbzPhYsqht1FXx3DIecGye4M2l2MorWvaycmwIp0B+OV6sOdcUc/o13ahFour3764lK7C/fYFe2pwRe5UleJEOd+j8lI8ImLgwM9Aclie4v3mjAOAmaxW1m668X9dOtuyQ== root@nixos

ekman/c0/ssh_host_key.d/c0-11.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSuaHY3hT//NqibrlVTpe4rBOixEVENAS4XG7cfM0pGZlVeRH1ebTki849pdpQ93j+r4Am4kTLYpIq5RHL+Pzpvp7FlY6DgeS/OCtyXNIC3gGe9xJU2khxRWkbYy7PXnjcFY19djy9Tkdgd9oIJqGYc190t9PPnwhG2TGo6LT6t0kTh0LEC3culgU9EAFAHkPaqnGjs5ElRknOqZ1M3jzLMKuL3+ScKRk33q+jb0+Gw/4Gl9fv/BPWQclP7E4PQaddc9oe/SqMN2lXMVV93n+QW/bImBtCAT6873h6wjDlKUmXu2WZPafiDjavbKLpgMuUSXzq9COxHGYN7cbqJENskzxz0awvRE5cBOL85qfZwkdN2uaKbdjRWp+EEcAWdLxWX/QlUS9PruQL1bVNe956nzyqsF3BH1f0M/nLuOv3ArJ0qGS1HgjJT5doVAUl76TqyNy37WZdIDVBFtsmz9XKZfei/1QzY0CKAFw8ZLKb+w68b19A8zrFNwH1W+lWZwxFGmlmn7kj0aoWYitQiDQ74eu0EJDtHgJHvYIgm4gmkXVLwMPPmg+GYJ1pVvh3ychWsZrHOoIEigZP8t/6vAZSFI7DWbMTXfn/aYSuM1qHhe1L2SqqjfdPwbf0EZjlUmj7C5JPv3a3QsGmUPdeugOLs3ow14iWH9k/NRTYzEUmDQ== root@nixos

ekman/c0/ssh_host_key.d/c0-12.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEniX5RkR0XcPBj+hZd0sC/1RdQhD/P6rvET55XWmGfkfnfhkMi5vHCv976dTk+H/oee1l/J2cfl6VKQpWGneiiH4ZRD6nqM7Du9AChWYFqP05x4vMag6RYO84qWbIKOE0sJYe52c2Q1TA3bZhaZZIojww/z4En1QDGsM3Gt0NqbyiLKlp5C3AVB+mF47vVb2eR32cnYmwa1Ab0ylQeLnVDtMg2QdXBpAQk5HbwXpyfzmH7VJnKVmF1cGei8LaeXB0M/EbbZMQJCXjrBpPQS3JfMPyFYfd7ok3vVvQiukkzRX+1Y7rWbDw20DHC7cQxFQLUU0cwSU2H6Kl+jFbMA40aQw9cZqUIet500p7SGK/32bZJxcV25XLWFBTAVD96mJhK3WPzeubI8tPmFI0Me7H9AMFgO51vcxqqVFzynF393k+DxGsMewWPJyrnar609j2CLRYjLiEDOjRD8Gywwo5QtIxmCMlaJhAzaJVr125Cp4JIxzKFqPfDsi9SmLfdLMlczQjQAvF0kS55laMXBxgM7ZHX1ihKq6bOnx4Lc1cKTeiLVH84srkhmHgY/G+m/eryQaY7feEVmqwbQovwEnCb+trhttibnOIilbTTVH71qE1rZL+hzgeUwL7jheCPzUQAWzdPvLLDYJq0yjGugHljRgtq7q1zFqrLGIaEgGjcw== root@nixos

ekman/c0/ssh_host_key.d/c0-13.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4wc7cxQGddBOqEbYRIo61cOsil//rkK2OKcuBEVXm4MlkYc7ZZIdSs34YAloPnuioTs1csnsGCGkUA1z4zLHIs/o/wLn1pTmA3ZYVevdaFl6CCDBgO/Tv/KXr/g3hq3Dhv9+2mzbu0Zc+44GBH+NrP77DsJAQiEkDH31U8MzqjMb0pzDbF2qI6GBa0IQPORevbJm3MT61MXvxBResYFDZhmgxA6HUokiUsUm0JImgDctwKAYZyzB9p19rZ9aPJxtnvQzh0EuBQc0Cfi8mlJ08DHFVK1F8/iQ98j0eOkTQuA+cqUBc2r0S5Ea0rK++qfEj0KsZy1av9nYlirUg9xNaWd9m/gMf9uMqY2eK3ly3+eSap8u91XIqm9NjFnLhjw7MXUmA+rXzU5gyfpcBu5pZwRQIMs7y5P42sll/kOhg9/VOmdnQqecvUm2ro10fQS8nRthLwVwHNsPFr5JSe36aKbF4bQIYZCBPmz8nLs5THdX3ONREfBRk3T0m5Tm19RvxWSyuaxJlvpTguZ25xk5J+1Qw6jocSsg75u+lP0X/Ty9Q7yCoEbNnLiQXpIU3IAl7RcLk9Edzj4xcpt5QLn6yOdwjn0qrYzke5lClX0l9btixYCyT6r6Z6NvKwJQBJHSDG/I74Brwt5Tr1SRpI6EvywF3rKKL3m9eKVGlrrNt6Q== root@nixos

ekman/c0/ssh_host_key.d/c0-14.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCkogj7O965v6DYNCld411vyUpsAL8+6hbVT2fjC1RZVyR01AW/+Mvle6aM90XpCMgwUCxgZIULNG4waFlCxXt5IGbOO3/vkHdznK9TUNBJllg69KAdp9nAePyZIczzxRXGh5lzu9RAE7Dywz8hkgELwHokGq8BBQ620/prMdlj4GRsSk3A3hdvbC5XuhwL+KJjrSecELjcvDtiURKDgkDglg7b3bahVais0meaMjkfYL4w1nm4AHnwDlriP6y3hJwBH3odJ1MdJmZBnS0Ebf/wYl2vL0kNACaQGh1+gzdhw2zviv7rW9Wuir07h6CuAQ1SV/b92lzk2Zekaeixf/6I1YNHWiQhfEVOZG/zibm8LdidJxUoLCxyO0h2nBGI3iuPgzUzAmrDSG8nSGNMJw3GU6ub4nj8s+6ZxV7nZ8omozqm6CP9Dw4H4iSRF1CVS3ePH44Zssgd7sbaPG5w4NAniKLRiTTwFfLYyyAhm7APHNrunFPX9SD2H2rJiqJ+16e4rDiBFjoQz5AO/+za+ITSNWZX7BLZM2Swo7jvrZWXYQQGhRj9r3Lzw6zZol+waC+uU4i5JBvQk4ksSNWkTAl/i2GpiUdy94ksBRSZCUWA6gQ49M8iftFNsLsT4KkuIpgV06zHSHKs/9NNovwiINiQo+zGBNNy0eGZTOXHIV/6nw== root@nixos

ekman/c0/ssh_host_key.d/c0-15.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVoWZnPzLuLfiTH82RIAweOqGfdZvKJe2LkaD39rPFr8w5gEIIYmorquYndpSynMH9P4N/HZZ0/kvyjFps4i6uZrJ6gBIZkFCayN2ljyf4/7kvIGlSQQ6c7a1Bty9oFIZYbsHFNNbCUqu8IF1aT3PBoiAjLZFkQlAaDUanIHb34XgDvaweU+s88QG0q7f7MN8eUlBkY4l34pDS2P28xfhbUl5xkDU5/cQXwZxdETsLgwPweXuCDQ58ZqYwqHUrp7GNfJ5QF9CnqTClrRKxfxJeUcDkvBnTWAvIAzMheWouBypyEguuaZDmvp80ylNKxkin18LFC61wOVHK6/1tDRp45CJ3Vhb9T+ch0ys0k3Qg/ebyCkKx1mOuV6waMq7imjWN4sQh7p6BjbrHuxFilczfv87F8p7TlzBMGDh8+ffL7G0ude578kEf05dHZT1M5o859l7iANgPYgEeaAocnZhwWy/0AYSG/zZeEVNCRS7YO0mTOwzPNmf1DhxArX9DrVPODwc7iHRVEy4KnqW+XcP8TcLQXOlqGEIIa+iE/AJlm2siqreoaUNsl9Gh48aiTX/90uzDnnIsMkNRDD05h0JwI1I7itAcTJVs6BX68CyIhD0VbNILsFqTGOHnJWiBR8z+79XnZwheZPM7xhv0n1mTvmi3syB8kZq6sGL6q7zTmw== root@nixos

ekman/c0/ssh_host_key.d/c0-16.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClq1v4N8cIxay8hXNOwU4ulrnf7Yyvnyt5reijIfOUfMiGskgi//kAdf5LyaQjyoZAApABTATVU/vIKhc/D2x96h1VJc1j+i9UmlUuawf7r5d5vcfar8Mc5T9cvXHOs/AWsjlH3vp5Kxac/A1+uk5VCGcML71iBRov3bZaM8RT2s+hzt5hu0RvclLPYuzPafKdsMJQg7dw8Y6pb7R45+8729ItKMHcK54IUxiOjLMbNCkHyUy6m0pPBTfj8G2PnoFkFdg77O4lVQnptyFU9wxSa76RLEqQp1CftaNPzxF6hD3VgGjIKSuiRz30A7J37lWMur6H5185NjWefDUyADa8JY5blb0BAP+csOrGxv39Ghh0wchE9I6Lozb+wGKphk4Esbq0wtKCsph3AZptyO3WCZeajQK5sZCawjOZUY0/t0cvtDefreDg0Db/cw1M6vc1fDgfIrHQ7Yq+y9QQmrClW+1b6pcv5DBKiipa9fUOZNkFcW+do5RKqr52IkuhGcG0pb5mbiGSBIb5EqgoEqBi2o5E5RQ8nWf9tF3xCrksCOjAnu5uUUoxPi14NC7z7p6fmwD3xvA/K3BZYkz1SrsccDcpNCSagPvxzdNTfBiW0USIMmWLQQ0lW7j8GjMGK6B0usNApj1WkiSHuzHNMQLVARaZGANXi/T8JfQoglglzw== root@nixos

ekman/c0/ssh_host_key.d/c0-17.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7wnYBwAdcp5+y7QOKzjd2apRv+Nvquy+qr3jVBeiQgbGsjlpGJBRsGv46QhSbB9QLuFuCUluuvilYbS9/ubm+BtRCrFViAHwFx+QhbLYlAXJPLGfXLxJZDSxbM6c9L+N+GAAfhiCfdhUVB39G33v5dd/VD3J4F/xC7lkPJU59pGokqe9dGrDta33pemSzR9QV9RuV4ZxJG1ld1B2L6TjSkAPTb79PPvMw8LnTViNffmHKp5qvwgIfwRvMApG5KnHw720pPncTW4ejY+utSbP4DcwD2qayKbxnsYDpimMDQxR3id6meZa5LcilxExmz/ig7Fpc3TB3wIpT4ha6VVu2FhRThKR5zD1Qif8BaKy5eAmqcWKQm6M9W56LmUdn/OHpzmaSv47UgtJ+1JG4LqGSaShs8kfVWtaqmev6vvnHBZMm7ni3xyROTSWnjJHLDQJbCYb1PBbct2Dod6MuU54CJUuxw0k+hCJ1QMz1ibme3+LauXVuLJ2JrnAoz7x2CcUv9e5MVx0H4HpWTCxi7cNBO5bbeuxMwHbl3JrZ58LBK9zZYl+nr2F7CZqLyoMimQuVpXT6Aw0k/nfYMAXjUJ3qbBldiGsRgpgbvL27nJK0EhQ8Kd36QhHfdJKhiAIPV+06WVWhPllaqay4OgZIQM1aB83eHg/OUxbOWEu89d7w/w== root@nixos

ekman/c0/ssh_host_key.d/c0-18.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/a5Q1Y8hta/v8W/cM45o6UimPLfSzz6NM+WaNAYbwnLoz2VVowHdjyndajmH1CMK/cTDRiVSrTkky48f3JNKzCMjmwVA6PjWmRxF0tqcitW9hzFB+L10+fgBsZu7MUMhuWfTtekIgnRjqNZIA3q5PGz9JrMHsRem+RGmDayaNSAkOZ1ToGSS7pnGm06qAswLJaK15wnupw5x9NYO1RXq2Dy2pX68e4urA+h9X9tqDfb6kQP3/9gb+q6BpiGUwUHW5x0HK/LwNqThKV7ZwzgMHqz4rq5TieGemTw8LKcDjj3BWQTDt4SlZ2gux49C7z1vBipHtxKHIdZH9AXYPZCkjgI/OVVJvxkbgMM+3FSr3j/KvTOdL5ycJvGBsF4dJnRTTpay7OmnksQ62+vwNO2dvDnhAgVntrr9Fb1ahLnTN7dOf6BlkUbz+vtYby0gcjr5sW8b4duGpJThDDkAGLUKkx1z94/bpPQpUb1ajXPizO7jjJCK20zTXWG2uiOdXWHmZ3YVoJ/qSOvtUAX2RR2FgJzHqnDnHtmLiMc9OkZ0eJwo6ltBqGHPNXCyBwHAa8eSH2Af+10vfzJ88J8MGD88DV7SYWbJxuB+KMAnDX04UfrvqLnRADzbFuiG8locorXFE90wEZTz/nFWsM+Q/XPA+M1GaO6fK9SyN45LgApKARQ== root@nfs1

ekman/c0/ssh_host_key.d/c0-2.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwC5iT7ZRgapwdzsgmFEkOETikSjDAX/z5peDh5G5rKSoakmSqde3tRJ5fRk1vptW/LkToZT+wU7F6PBQRuI/OyOBIud5Vubr+lqctHy2AfvI1x8SdMRPAIpAhklISKsV9iQH+oMgl7ND4gfLblBbLksQ8iCojlsXcoTwjlt8HYnGNJV/kC5leIIibKDzyMjC0F0jlMiWbK07WllmT0ecNssSV5HNVbY29lW9Fv6J25gp0i025lpB9GC0PiIOoxMVzYRNJCEi1nNPNipuNFOx3XnwRjbVvM1KEP4JqavgXVFXUr2at0C0x+hcKtQ7LpdyvmmAywTMTTljhdU/0jNw8pk57HhpJ1rXQZm4SIqL7gdwDy91KLM3qE1MjzocWZw4YBNDfeNSS6qalxowuut354x8UItEjIwy1uofRF8RKB7LQYqKbXHWCZeA+3rNA4E+QIuvbwX8HObMS9JXzTUSUL7r53rAFNVifYSey6u2iZ+PT+jKkXKOCOE6fuCy/MPVaiJVwtd0wVsgC4HH6CvrJf1dwlI5Av4B8LrnZNgpjog+aqgDHjRRY+3ys5Iwt+DqHHLGiHyoM/mbF0Hl5GQ3SLdfhn27zLndLnBg6uBSIRmssn0Vech4gndy4nUNZpWyKWoy4lt5aVLOoojD9G3Kp5XfQVOlJToU9D73n01hclw== root@nixos

ekman/c0/ssh_host_key.d/c0-3.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+rokHymxlz8S5jYU6Vamp1TVX+M/rAHGflMWwpAYND7WSt1LkCICHbgAWx9lCXGDEidMuajENtu62tW2zar3cXRrk/p5SdwCb+NiztQ4uGTzGIMxYA52Cjy6KyMi7pG9C6/91WyHFy9aKPFjbA1c2FPgf7eqIYg9B3uBu75yCWCuZ3ZRs+al3R8Tbqo4J7sSanxWRpdc0t7eU0U/rMeTi41SXmNF82smHaQ7iOnWajHAWwybmNpWQkiFpvyahzct0No606oODFl/SttI8R9CCJpvlEeT/Q+nj0hgMciw3J604EJaaLgaYfUcJg/V4F3fe8eGDiOzBYBYrYguNv/XVUW0llP2K+SG0zi/3luQTMghpd+fsGr49cdwFrJWXJhqvE2O5QlQ3mWw+pwlUY7Qgs2usGbQK04VGpJlFiSkso4pythuNMFB37YZoA57+RozsjpeiVmL37xcq6jfsd7ljvjbUICCGLaaJN7p1c3OQSw1pma1R3G14FvefhUQe+F2DLyLfvFq476KN0TSUU42tw9iKfCbP3geZp7UkqpyONerViP9VOa4CQclyuaZ5Yg1x4v7zmfQ6J8J1+sqSwEUe7UnaPuKM2XklViT0AUy33EKIkzGe91aBOwcQMaA7fYHYf1Jvf5lYI5MiA56hkV8f9KVZsKGD9advK+WnsdyUCQ== root@nixos

ekman/c0/ssh_host_key.d/c0-4.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCmlNP7HKPGYtciCLRL6KyPs1JPF/ehYWuMmeiJrVxdq3YSrfAddB/6cdHMOw/RLukdyLdIwvRfGhy6QZq9BbKBFSJKypBGNe3KTTT7zd/qxNKEbQhJ8m43NtBbOUHBPQ49zE2eunGwMYKL2EW/7ndbtzQOTTrpuZTsGXB6W6Rn/ZnYZC9wXmOyn5XJKleOhAlor0wIyORdAPwdY0s9LdJ6lJUCUqxstFecuqIsgnaMUBbzyTl09e2RbtMduEjRH/B7CHBxj8P4oVlBGTVStOAeeqMudrJeXvRssMB1BX9Tu92CVYnr2FsUazJ+s1R0dHqdXFuWuR2oddw3y1Et0RyAoNmgiku+ZwIbqousNZx8UxrGlMII1fi8RdNqmMZE+8LhC3yhztYQloZ8Ak8J8SpQE7av1gvlHKCMD5bxtBD4mNfj3SSdH9TnF8gu1x6zT0A4LdL+PqK6E+2zZlc54DfLT3SwS214kFUtgM4t9VhSC2E+EHGCgZ91U6xmsd4zVGMaGvzRXcyH0rROzCKsP2M6rVssOER1ePXEZUIC+62PC9IY9tM8lFDH2AB/JF/xdLZYtO9pdWG6d6DOKSpiP2qlMILIX9/qivHTCySnyP3lBtJ7THF8zsJ5sTUO53/Grd0mSLSqTm3T12PIwABPWwiEXKMD2pIszUhu44KRAF0sJQ== root@nixos

ekman/c0/ssh_host_key.d/c0-5.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDmF23dxQHzmN1rVlDkHelCBK48NFc646LFZyB5Gn8cphpur9wV26iOxWOXct+ITU9p6fWmXDko9zCrCWcsF6Ib3rBzbCHKMzF6Vuj0UZg3XP4IUrOdw+Cd5L8GTVoAzskXL0EM+upKPeXFBob8OS3F9Ax0X9ZT66vqY5QGuZWtUikWnakTxbl7JFj+Hgx5/ZzS5cH2e/9NoKkl7wGSkUp0rqjfIEimXI6SNl98kJUtCWEWyUvon4yLuJ4X3+4wrjwdgznspOWgCt0XnTnmak5ADrbz+VuXnDR8TB4AJNfhFySW6E4ZRSrGpZrt1FfaUBxTCmq2Zq/t1chA0Z6z+/k2vCjd9CXodAUalgVmJZ7EQZdSSo8nXA4P5Y8I6TMesTKlBVxMlxqsQfwve+xaPV0/+QSXCvpdRep3+QDVZXaOO4N7UU2xeQR38cNHNzovMy1z5439nq4p5uoA3CxWLL/z6vsZQ3s3xXdvlvPudGWmTW2c0FGKS8ALSWUKLibvPHsEEdIwLJ6FtUs0Vp3QtBKdq/NmN05gZJEOrnfKr8JpvVl9vzRAc8wCYsTyujHTkkYb/E+fb7VsEngbD8iu6PiRgeOui/t0QkTl1ij+SD0YnDXrh61YhQjuvdogJq47gtcSbCE8Q2rkzDpDIEOJNZP7uZINyl9VIRhBb6jt7CC/kw== root@nixos

ekman/c0/ssh_host_key.d/c0-6.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4dNgyC5n1PQ9hp+o1HuTVt7MXkt5Y7wOFAmR+dzWWQ5X/3p7umJzl/I0955CuaXPTb6E6Oqi8WQAlqlBhk9w8qoqoczXsHbqSbnoWSk82VgvHxPv2+5iWvth/FvuTEiM43tt2AhDxG/8TobR2eFD2/2h/T7OpJUcZ7aHT6BZnN5IS/P6MgarOUootDNx9zn3BXFM4Pg8lnB9dnKm4iO7Ju9kk9s2eWzvk37QpNE+H/h58DOMExUcrDzftXNrVbUeZjw7Ho9MER+fdbucrcdKxFH/NiwXlgRSRd+Z5Y+IG+WJdkdJw1hW47MsGRGH+yEq/woE4YvzdcqqqLh9m2a2ucQkk1pe3xpYXCxwS8aVcIDzlja7rGaa2nSBz+Lkp+G5awT2Sg4ob7vQYIN2X6p5zYsqLfWdU5GloWgjEB/zvADjF7qKeEdFFv/eXzaiwnXSoOofcCZTW0qAldK7cdO/esDUfQGIsti8qk1adXUGXG52UhV0p+rMtloW2AAJVFsUiUWvp2Ue3i+p+nAC17aDMOq1UrR60/n1FcjM7mbwAjLgwQ40tLjc54HoMVg92lMTVgJgk+gBYKFsfrI3nbSmvUiRFZDWzPFSno9WtFasWrNCiQ9srmfP0Ud6ro0QCtnvSxFMuXiVufaUlRka0SLlZbA8N3mLrZ0M+7rWy5syLSw== root@nixos

ekman/c0/ssh_host_key.d/c0-7.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBcImgez4RkX1TuqDTk/AUugMqSfsIjdWmEyJRgGw0ZqYSiz9R0yHVM9E6tFIXkGw73RsF1qVQorcw3ZRrxiYLtJHRyBnX0/6IHFwT5w1TezH7LMJmId3ys+LEcpqnCdVaPo2RVsq3RJhnM4dImVC36UUJj8TiOzhZ03lTP7TCPABxvwzn//+T5EjyKuK7acFU4ts4ii7B28dWErCVjsq29uWxKXopfER+45DjhiilQgJ5R15voBNvvFCR4bfnVrsyBwHXXRohAOU1yJYpsUDkXMDY2DIG41LqpP8lg8IyPF36Vwt4w8vmK6rpxBvy4SKg4TIoxUK3eqODnrgN15/2yDTRWWF8u6rpEopWRQEUlKqZvHnXFWuSN5TGUBeOrjqom92udfGrW0l6Yn6iUEt2n+OkPxRuDAGGwjRf4oV2RSqbhbIKfctJg0b+8VhrecNow2zyNlRvfLiQvKotQEDolvfnOYyIWujsjUWttl5a0eoJBm40QgNvWHJWfuaheMhVfEWcfoqgkvZDfBeE11Z/1eogCXMWELJaX/gevE7I+0veu8myzmRwXQOEbk+Ya8tVCSm6YrZjp7flUDRNMW9bZswEdS/ZjvIlK5RgDCAakBoU4MY/PZipYujFoT4AQTNiV3AfuakktXY8dORUHDOcD8zmdbE6B/6t7d1SrHIOow== root@nixos

ekman/c0/ssh_host_key.d/c0-8.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVFD6R1Mwz0+SqojCUcwLlOm6fKarAPEOB1Ysjo8daGmwx/zfzeHLkzl2RpA8NpSAHheBQJxt+x108wZRZjy+EWYzc/sOaui7lFvZVxxB2ewv1yAoBXUl82O1D2i2ZaaAoLa4G6dZnSfdd0THJNqMIoCCThDaNI3jGcTOzHivVb1XWpdM1DIQtbTcwOJ3Q0UZTiNi5eL+7wlpIKE6/IzkSyTgyYi463CEeR8YXCaILdRweJf5lFem+ahD6uUBe8yuNktpnE8wzQXSHBlNrI3C4Kk9uQicDr1uThexBX8/37hg1V7Wo/AxBQcWlumUCgtnlC53V63XvcRjfB9zrQSbHgZcDnlgGgvMPYillLNCGcFdny1ZbpWpQwlt6kZgHmB3v5m+i63iaCYaAzAoGfp/M/f3mrrQF9GYdsFtngZsGsTxv1Rr9Nmef05V3CewKPaLw13XwPbH+Muvzdx00P9W7P2VmryQXYmAVs4iczUIdRGaTDGJIou4qycRHMAfC6aYEvR2s0+oTzLCMdYgjkVDHxJDkrgSP0Z8++bJa2qPASrTDnT7Hpghup7l1NspJpiZX3vIBLOi+0cBfoRZuLrQ45dDCrRdCySDvv5PfngACqNWt8PCHBeQAHS81d7+xb0e92VfxtZEABTwHlWePMuRmSiSXBi/UGdruFsFm0n0qPQ== root@nixos

ekman/c0/ssh_host_key.d/c0-9.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC9X4NFHsCryXjQOiUrxcTNOx0JpPiGViEuvEhB23C0270bwbxlijaIsTrAxB8lfOaKHGvzXfMBX0lV9QGXfIPTXIr0hbmOy6xfJq6lRQwbiRA6YS4T1yic25mx47F0ODqaW3mTykpIE1qpz4Wzl3M1ZLbbT1wcvWTFJ6MCY74OR1vNrtR7FgVFrh7q7iT1+a8Qzr5zG39yphO9aYiBKNLle820oH1UxsJ3W5e38ZI/huJ2w1Y0TY6eghedEZvGIhm0qDJHlc8EmGterCUjCLZKcZgHNGIC6FnT3n8HGwsTtYHc2mRW3W4BK1iIpncYkDSX71xY4ev13Y7UbD6rVBWqC+eKD/3pm+HlvXxk8tvDZfcq4ngSEIoU7d5FeLgE2hidE92zLaKdXpJ0PaLo1Gz3M0CY3TeaCWZBSnToGY106ynUipoGJ29cDaQWVlYkWAdC7d+3eR1yjpT/98PNx08m8fVs/PT30/LkR6kpUpCB1GBdOu1/Q4DFeGtY1BFNibzoWyk6ByagOMdEfRzPn99J4Vd0zlzwyzFkJTlI7OHeVwBFPHW0joGm3x3MGbRmFQBBIA3C9b9a6mFseQpMX/T++v0h98x7mvsMx2Nf+1O3uRx3yLp47GfyatsKB8oUSi5dI7S3cAyAZIB8E8ot4dkdounPzA2L8ipnzwdEjpqwLw== root@nixos

ekman/c0/ssh_host_key.d/fix.sh

@@ -0,0 +1,12 @@
+mv c0-8.pub  c0-6.pub.tmp  
+mv c0-6.pub  c0-7.pub.tmp  
+mv c0-7.pub  c0-8.pub.tmp  
+mv c0-15.pub c0-10.pub.tmp 
+mv c0-14.pub c0-12.pub.tmp 
+mv c0-12.pub c0-14.pub.tmp 
+mv c0-10.pub c0-15.pub.tmp 
+
+for i in *.tmp; do
+  mv $i $(basename $i .tmp)
+done
+  

ekman/c1/connauthfile

@@ -0,0 +1 @@
+莵q丘

ekman/c1/default.nix

@@ -0,0 +1,129 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
+  # }) {};
+  # pkgs = import <nixpkgs> {};
+
+  nodes = import ./nodes.nix;
+
+  mkCompute =
+    host:
+    let
+      hw = ./hardware-configuration.d + "/${host.name}.nix";
+    in
+    {
+      "${host.name}" = {
+        deployment.tags = [
+          "compute"
+          "c1"
+          "cluster"
+        ];
+        deployment.targetHost = host.address;
+
+        cluster = {
+          compute = true;
+          k8sNode = true;
+          mounts = {
+            rdma.enable = false;
+            gbe100.enable = true;
+            automount.enable = true;
+            users = true;
+            opt = true;
+            work = true;
+            data = true;
+            ceph = true;
+            backup = false;
+          };
+        };
+
+        features = {
+          host = {
+            name = host.name;
+            address = host.address;
+          };
+          os.networkmanager.enable = false;
+          os.externalInterface = "eno33np0";
+          hpc.computeNode = true;
+        };
+
+        # services.udev.extraRules = ''
+        #   KERNEL=="ibp1s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+        # '';
+
+        networking = {
+          useNetworkd = true;
+          hostName = host.name;
+          useDHCP = false;
+        };
+
+        # systemd.services.systemd-networkd-wait-online.enable =  false;
+        # systemd.network.wait-online.ignoredInterfaces = [ "ibp1s0" ];
+        systemd.network = {
+          # wait-online.enable = false;
+          networks = {
+            "40-${host.iface}" = {
+              DHCP = "no";
+              matchConfig.Name = host.iface;
+              address = [ "${host.address}/24" ];
+              networkConfig = {
+                DNSDefaultRoute = true;
+              };
+              routes = [
+                { Gateway = "10.255.241.1"; }
+                {
+                  Destination = "172.16.239.0/24";
+                  Gateway = "10.255.241.210";
+                }
+                {
+                  Destination = "10.255.242.0/24";
+                  Gateway = "10.255.241.100";
+                }
+              ];
+            };
+            "42-enp65s0np0" = {
+              DHCP = "no";
+              matchConfig.Name = "enp65s0np0 ";
+              address = [ "${host.gbe100}/24" ];
+            };
+          };
+        };
+
+        # boot.kernel.sysctl = {
+        #    "net.ipv4.tcp_timestamps" = 0;
+        #    "net.ipv4.tcp_sack" = 1;
+        #    "net.core.netdev_max_backlog" = 250000;
+        #    "net.core.rmem_max" = 4194304;
+        #    "net.core.wmem_max" = 4194304;
+        #    "net.core.rmem_default" = 4194304;
+        #    "net.core.wmem_default" = 4194304;
+        #    "net.core.optmem_max" = 4194304;
+        #    "net.ipv4.tcp_rmem" = "4096 87380 4194304";
+        #    "net.ipv4.tcp_wmem" = "4096 65536 4194304";
+        #    "net.ipv4.tcp_low_latency" = 1;
+        #    "net.ipv4.tcp_adv_win_scale" = 1;
+        # };
+
+        boot.kernelParams = [
+          "console=tty0"
+          "console=ttyS0,115200"
+        ];
+        systemd.services."serial-getty@ttyS0" = {
+          enable = true;
+          wantedBy = [ "getty.target" ];
+          serviceConfig.Restart = "always";
+        };
+
+        imports = [
+          hw
+          ../default.nix
+          ../mounts.nix
+        ];
+      };
+    };
+in
+builtins.foldl' (a: n: a // mkCompute n) { } nodes

ekman/c1/hardware-configuration.d/c1-1.nix

@@ -0,0 +1,29 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+  # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_7;
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/d89e1496-fda1-4de0-b2cc-474967b04402";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/A51A-1F4D";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-2.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/f0826ad5-8a4e-427d-98d3-5afa44440993";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/21A6-D34C";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-3.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/3af91585-8079-420d-acdf-f60b94d3cfff";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3590-199A";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-4.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/84cc9cea-08eb-4b54-8ca3-2aa5c1300a92";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/882D-A342";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-5.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/d23386a8-0ca8-4871-b662-decf2b24f4d7";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/AF8A-DEFE";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-6.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/2f468316-5832-4684-866d-2e92b08fb68b";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/76FE-F657";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-7.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/37944fce-07ca-492f-906a-620a37e7e1b3";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/2F51-EC20";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/hardware-configuration.d/c1-8.nix

@@ -0,0 +1,28 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/12477966-c6c5-47c6-afdc-35fa7e57e837";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/DB96-7453";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+}

ekman/c1/kernel.nix

@@ -0,0 +1,46 @@
+{pkgs, lib, stdenv, fetchurl, config, kernel ? pkgs.linux, ...}:
+let
+  i40e =
+    stdenv.mkDerivation rec {
+      name = "i40e-${version}-${kernel.version}";
+      version = "2.13.10";
+
+      src = pkgs.fetchFromGitHub {
+        owner = "dmarion";
+        repo = "i40e";
+        rev = "7228a7c3b362c3170baa2f9a9c6870a900e78dbd";
+        sha256 = "087kvq9wrc1iw6vig8cqcx7cb6346wx8qxzb85c3n8638vq1vrxr";
+      };
+
+      hardeningDisable = [ "pic" ];
+
+      configurePhase = ''
+        cd src
+        kernel_version=${kernel.modDirVersion}
+        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' Makefile
+        sed -i -e 's|/lib/modules|${kernel.dev}/lib/modules|' common.mk
+        export makeFlags="BUILD_KERNEL=$kernel_version"
+      '';
+
+      installPhase = ''
+        install -v -D -m 644 i40e.ko "$out/lib/modules/$kernel_version/kernel/drivers/net/i40e/i40e2.ko"
+      '';
+
+      dontStrip = true;
+
+      enableParallelBuilding = true;
+
+      meta = {
+        description = "Linux kernel drivers for Intel Ethernet adapters and LOMs (LAN On Motherboard)";
+        homepage = https://github.com/dmarion/i40e;
+        license = lib.licenses.gpl2;
+      };
+    };
+in
+{
+  i40e2 = i40e;
+  overlay = self: super: {
+    # linuxPackages_5_4 = super.linuxPackages_5_4 // { inherit i40e; };
+  };
+}
+

ekman/c1/nodes.nix

@@ -0,0 +1,13 @@
+with builtins;
+let nodes = genList (n: n + 1) 8; in
+map (n: (
+  rec {
+    idx = 120 + n;
+    iface = "eno33np0";
+    name = "c1-${toString n}";
+    # target = "10.255.241.${toString (idx + 100)}";
+    address = "10.255.241.${toString idx}";
+    ipoib = "10.255.243.${toString idx}";
+    gbe100 = "10.255.244.${toString idx}";
+    pubkey = ./. +  "/ssh_host_key.d/c1-${toString n}.pub";
+  })) nodes

ekman/c1/ssh_host_key.d/c1-1.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBEV8wEkeGUOs6umhdeOKYnVlYlta2rOCZSoezvu+bZ4 root@nixos

ekman/c1/ssh_host_key.d/c1-2.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDIsqJqWWftoiyiOOHnIuHYqbsaOg4AbKNm80wpjH2E8 root@nixos

ekman/c1/ssh_host_key.d/c1-3.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwwu4RJJhKo8s2Mtmpdvxs02d4IwrmS9cTpfAOQ2YkI root@nixos

ekman/c1/ssh_host_key.d/c1-4.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQFQqo2vcj2eYaH7nwdEzgCPme+7g3Db+s16KbQHzLI root@nixos

ekman/c1/ssh_host_key.d/c1-5.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImItUKMSygMY/1ZNsyGkyfywyngEDgZ7TxM63UwG1VH root@nixos

ekman/c1/ssh_host_key.d/c1-6.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBq+v2bBhQieqfXmtmGYb/9mOv9oc88zerRkkpGpoc0x root@nixos

ekman/c1/ssh_host_key.d/c1-7.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDme/xztp22wQOybtN2TVXMcn2QcVaXtRMp4AnPnzr2T root@nixos

ekman/c1/ssh_host_key.d/c1-8.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID1WxWnNg+L2+lQ3W2mD0/6kqwuCUOEJImvWnQYLzdUB root@nixos

ekman/default.nix

@@ -0,0 +1,360 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+with lib;
+let
+  cfg = config.features.host;
+
+  computeNodes = import ./c0/nodes.nix ++ import ./c1/nodes.nix;
+
+  mkSANs = host: [
+    host.name
+    host.address
+    "127.0.0.1"
+  ];
+
+  configuration = {
+    system.autoUpgrade.enable = lib.mkForce false;
+
+    nixpkgs.overlays = [
+      (import ./overlays.nix)
+      (import ../modules/overrides/certmgr.nix)
+    ];
+
+    boot = {
+      loader.systemd-boot.enable = true;
+      loader.efi.canTouchEfiVariables = true;
+      # kernelPackages = pkgs.linuxKernel.packages.linux_6_9;
+      kernelModules = [
+        "ib_umad"
+        "ib_ipoib"
+        "ceph"
+      ];
+      # kernelParams = [
+      #   "console=ttyS0,115200"
+      #   "console=tty0"
+      # ];
+    };
+
+    services.resolved = {
+      enable = true;
+      dnssec = "false";
+      fallbackDns = [
+        "1.1.1.1"
+        "1.0.0.1"
+      ];
+      # domains = [ "ekman.tos.obx" "~." ];
+      extraConfig = ''
+        DNSStubListener=no # conflicts with dnsmasq and kubernetes dns
+        MulticastDNS=no
+        LLMNR=no
+      '';
+    };
+
+    console = {
+      font = "Lat2-Terminus16";
+      keyMap = "us";
+    };
+
+    i18n = {
+      defaultLocale = "en_US.UTF-8";
+      extraLocaleSettings = {
+        LC_CTYPE = "en_DK.UTF-8";
+        LC_TIME = "en_DK.UTF-8";
+        LC_PAPER = "en_DK.UTF-8";
+        LC_NAME = "en_DK.UTF-8";
+        LC_ADDRESS = "en_DK.UTF-8";
+        LC_TELEPHONE = "en_DK.UTF-8";
+        LC_MEASUREMENT = "en_DK.UTF-8";
+        LC_IDENTIFICATION = "en_DK.UTF-8";
+      };
+    };
+
+    time.timeZone = "Europe/Oslo";
+
+    environment.etc = {
+      "aliases" = {
+        text = ''
+          root: jonas.juselius@oceanbox.io
+        '';
+        mode = "0644";
+      };
+    };
+
+    features = {
+      os = {
+        # boot.uefi = true;
+        adminAuthorizedKeys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas"
+          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
+          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2tox0uyFGfU1zPNU6yAVSoGOUkeU959aiTMrqu1U9MCCOP2o4IhZIlRpZ08XVnUU/AhycCUF4HgGqdcco8oIVX0P0Cn83KJoD/DOqAiz+1VwIUUV1ylrRdNqCgf4wnmLni3sUPHJdQnuq57+pzDDjHMr9CcBL2KzOHD/QanfR+jZmv9K3OS5oDcWquSCziXkpbkWQURPactmtyzGK2FRRxONZgYrB8gRTDstlWQg/t6GHNVelzuJ7SEf+t8pk/S2e/XAvfZyRJhrVJ35iZKpmxkIn5v0g1Z+z0yX/KRSAPRtNg9uM44cmto77MFx7iFs0CuleL3zHvRvZYW1ZnsKAiP07UkEK87luMpkTzFr9CSHJGpgk1RZYA3qidQti44n6NU9YRNhzO4v+KQE6XDqO80gZCJboSXr3fnYn/QHpPXzK5JcZNWmClyMURYj10qv9So3Fh0o3LV5GThA6JgN874vUywUZanPEdn8ePBcAsjLRzA4YBGEuvJCc6FELSuY2s+/pFba8NXQvrOdJKSRC0g5USQFfaWDln4Q4zZ1G5z76p1u6GtRWxvakkUQ0fze9KAW7msxeKaw+B7uMtyvCL8V2zEE8WKFP1sNyYEe7Sgp3RVfym2VPMNTZVhEImfM/3D+WbzfoJztnJvFKXeeMCcne4G8swyef3o1s3b+CvQ== Simen Kirkvik"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHVwcJOtx9YTWy+aD4xGbyPFLOdMN6NqY8wcfDtHczyT Stig Rune Jensen"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfgY468dPNpdXZCkD9jw1p2qA0+z56Wi/c1VYE+riki Stig Rune Jensen"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst"
+        ];
+        docker.enable = false;
+      };
+      cachix.enable = false;
+      monitoring.nodeExporter.enable = false;
+      mft.enable = true; # Mellanox MFT
+    };
+
+    networking = {
+      useDHCP = false;
+      domain = "ekman.tos.obx";
+      nameservers = [
+        "10.255.241.210"
+        "10.255.241.99"
+      ];
+      search = [ "ekman.tos.obx" ];
+      extraHosts = import ../hosts.nix + import ./hosts.nix;
+      firewall.extraCommands = ''
+        iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
+        iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
+        iptables -I INPUT -s 100.64.0.0/24 -j ACCEPT
+      '';
+    };
+
+    environment.variables = { };
+
+    # systemd.services."serial-getty@ttyS0".enable = true;
+    # environment.etc."beegfs/connauthfile" = {
+    #     source = ./connauthfile;
+    #     mode = "0400";
+    #     uid = 0;
+    #     gid = 0;
+    #   };
+
+    services.certmgr.validMin = "120h";
+    services.certmgr.renewInterval = "30m";
+
+    nix.settings = {
+      max-jobs = 32;
+      trusted-users = [ "@wheel" ];
+      substituters = [
+      ];
+    };
+
+    system.activationScripts = {
+      kraken-permissions.text = ''
+        chmod 755 /work/kraken
+      '';
+    };
+  };
+
+  slurm = {
+    features.hpc.slurm = {
+      enable = true;
+      client = true;
+      clusterName = "ekman";
+      slurmctldHosts = [
+        "ekman-manage(10.255.241.99)"
+      ];
+      dbdHost = "slurm-accounting";
+      mungeKey = ../munge.key;
+      jwtKey = ../jwt_hs256.key;
+      # slurmKey = ../slurm.key;
+      # pkey = "0x7666";
+      mailDomain = "oceanbox.io";
+      nodeName = [
+        "c0-[1-18] Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
+        "c1-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
+        "ekman Sockets=2 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=500000 State=UNKNOWN"
+        "ekman-manage Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=92000 TmpDisk=200000 State=UNKNOWN"
+        "fs-backup Sockets=2 CoresPerSocket=20 ThreadsPerCore=1 RealMemory=92000 TmpDisk=300000 State=UNKNOWN"
+      ];
+      partitionName = [
+        "batch Nodes=c0-[1-17] Default=YES MaxTime=INFINITE State=UP"
+        "ekman Nodes=ekman MaxTime=1:00:00 State=UP"
+        "short Nodes=c1-[1-8],c0-18 MaxTime=INFINITE State=UP"
+        "long Nodes=c1-[2-8],c0-18 MaxTime=INFINITE State=UP"
+        "stats Nodes=c1-[7-8] MaxTime=INFINITE State=UP"
+        "test Nodes=fs-backup MaxTime=INFINITE State=UP"
+      ];
+    };
+  };
+
+  compute = {
+    system.activationScripts = {
+      mkWorkDir.text = "mkdir -p /work";
+    };
+    cluster.slurm = true;
+    features = {
+      hpc = {
+        enable = true;
+      };
+    };
+  };
+
+  k8s-node = {
+    features = {
+      k8s = {
+        enable = true;
+        node.enable = true;
+        clusterName = "ekman";
+        initca = ./ca;
+        cidr = "10.100.0.0/16";
+        master = {
+          name = "ekman-manage";
+          address = "10.255.241.99";
+          # extraSANs = [
+          #     "frontend.oceanbox.io"
+          # ];
+        };
+        ingressNodes = [
+          "ekman-manage.oceanbox.io"
+          "ekman.oceanbox.io"
+        ];
+        fileserver = "fs-work";
+        charts = {
+          acme_email = "acme@oceanbox.io";
+        };
+      };
+    };
+
+    system.activationScripts = {
+      copyCaKey.text = "cp ${./ca}/ca-key.pem /var/lib/kubernetes/secrets";
+    };
+
+    services.kubernetes.kubelet.extraSANs = mkSANs {
+      name = cfg.name;
+      address = cfg.address;
+    };
+  };
+
+  shosts = {
+    environment.etc."ssh/shosts.equiv" = {
+      mode = "0644";
+      uid = 0;
+      gid = 0;
+      text = ''
+        10.255.241.80
+        10.255.241.90
+      ''
+      + builtins.foldl' (a: x: a + "${x.address}\n") "" computeNodes;
+    };
+
+    programs.ssh.knownHosts = {
+      ekman-manage = {
+        hostNames = [
+          "ekman-manage"
+          "ekman-manage.ekman.tos.obx"
+          "frontend.oceanbox.io"
+          "10.255.241.99"
+          "10.255.243.99"
+        ];
+        publicKeyFile = ./manage/ssh_host_key.pub;
+      };
+      ekman = {
+        hostNames = [
+          "ekman"
+          "ekman.ekman.tos.obx"
+          "ekman.oceanbox.io"
+          "10.255.241.100"
+          "10.255.243.100"
+        ];
+        publicKeyFile = ./login/ssh_host_key.pub;
+      };
+      fs-work = {
+        hostNames = [
+          "fs-work"
+          "fs-work.ekman.tos.obx"
+          "10.255.241.90"
+          "10.255.243.90"
+        ];
+        publicKeyFile = ./fs-work/ssh_host_key.pub;
+      };
+      fs-backup = {
+        hostNames = [
+          "fs-backup"
+          "fs-backup.ekman.tos.obx"
+          "10.255.241.80"
+          "10.255.243.80"
+        ];
+        publicKeyFile = ./fs-backup/ssh_host_key.pub;
+      };
+    }
+    // builtins.foldl' (
+      a: x:
+      let
+        n = toString x.idx;
+      in
+      a
+      // {
+        "${x.name}" = {
+          hostNames = [
+            "${x.name}"
+            "${x.name}.ekman.tos.obx"
+            "10.255.241.${n}"
+            "10.255.243.${n}"
+          ];
+          publicKeyFile = x.pubkey;
+        };
+      }
+    ) { } computeNodes;
+
+    environment.systemPackages = [
+      openssh-shosts
+      pkgs.inotify-tools
+      pkgs.ceph
+      pkgs.ceph-client
+    ];
+
+    security.wrappers = {
+      ssh-keysign = {
+        source = "${openssh-shosts}/libexec/ssh-keysign";
+        owner = "root";
+        group = "root";
+        permissions = "u+rs,g+rx,o+rx";
+      };
+    };
+
+    # Use nvd to get package diff before apply
+    system.activationScripts.system-diff = {
+      supportsDryActivation = true; # safe: only outputs to stdout
+      text = ''
+        export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
+        if [ -e /run/current-system ]; then
+          ${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
+        fi
+      '';
+    };
+
+  };
+
+  openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {
+    buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ];
+    doCheck = false; # the tests take hours
+  });
+
+in
+{
+  options.cluster = {
+    compute = mkEnableOption "Enable compute node configs";
+  };
+
+  options.cluster = {
+    k8sNode = mkEnableOption "Enable k8s node";
+  };
+
+  options.cluster = {
+    slurm = mkEnableOption "Enable slurm";
+  };
+
+  config = mkMerge [
+    configuration
+    shosts
+    (mkIf config.cluster.compute compute)
+    (mkIf config.cluster.k8sNode k8s-node)
+    (mkIf config.cluster.slurm slurm)
+  ];
+
+  imports = [
+    ../modules
+    ../nixos
+    ../users.nix
+  ];
+}

ekman/etcdCluster.nix

@@ -0,0 +1,13 @@
+{
+    enable = true;
+    existing = true;
+    nodes =
+      {
+        frontend = "10.255.241.99";
+        fs-work = "10.255.241.90";
+        fs-backup = "10.255.241.80";
+        # ekman = "10.255.241.100";
+      };
+}
+
+

ekman/fs-backup/default.nix

@@ -0,0 +1,235 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  name = "fs-backup";
+  address = "10.255.241.80";
+  etcdCluster = import ../etcdCluster.nix;
+in
+{
+  fs-backup =
+    { config, pkgs, ... }:
+    with pkgs;
+    {
+      deployment.tags = [
+        "fs"
+        "fs-backup"
+      ];
+      deployment.targetHost = address;
+      system.autoUpgrade.enable = lib.mkForce false;
+
+      systemd.targets = {
+        sleep.enable = false;
+        suspend.enable = false;
+        hibernate.enable = false;
+        hybrid-sleep.enable = false;
+      };
+
+      # services.udev.extraRules = ''
+      #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+      # '';
+      environment.systemPackages = with pkgs; [
+        rdma-core
+        hwloc
+        xfsprogs
+      ];
+
+      cluster = {
+        k8sNode = true;
+        slurm = true;
+        mounts = {
+          rdma.enable = false;
+          automount.enable = true;
+          users = true;
+          opt = true;
+          work = true;
+          data = true;
+          ceph = true;
+          backup = false;
+        };
+      };
+
+      features = {
+        host = {
+          inherit address;
+          inherit name;
+        };
+
+        os = {
+          networkmanager.enable = false;
+          externalInterface = "eno1";
+          nfs.enable = true;
+          nfs.exports = ''
+            /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+            /exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          '';
+        };
+
+        k8s = {
+          enable = true;
+          node.enable = true;
+          master.enable = false;
+          inherit etcdCluster;
+        };
+      };
+
+      systemd.services.rc-local = {
+        description = "rc.local script";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        path = [ "/run/current-system/sw/" ];
+        serviceConfig = {
+          Type = "oneshot";
+        };
+        script = ''
+          # if [ -e /sys/block/md126 ]; then
+          #     echo "deadline" > /sys/block/md126/queue/scheduler
+          #     # echo "4096" > /sys/block/md126/queue/nr_requests
+          #     echo "4096" > /sys/block/md126/queue/read_ahead_kb
+          #     echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
+          #     echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
+          # fi
+          grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
+          grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
+        '';
+      };
+
+      boot.kernel.sysctl = {
+        "vm.dirty_background_ratio" = 5;
+        "vm.dirty_ratio" = 10;
+        "vm.vfs_cache_pressure" = 50;
+        "vm.min_free_kbytes" = 262144;
+      };
+
+      networking = {
+        useNetworkd = true;
+        useDHCP = false;
+        hostName = name;
+        firewall = {
+          allowedTCPPorts = [ ];
+          allowedUDPPorts = [ ];
+          extraCommands = ''
+            # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
+          '';
+        };
+      };
+
+      systemd.network = {
+        networks."40-eno1" = {
+          DHCP = "no";
+          matchConfig.Name = "eno1";
+          address = [ "${address}/24" ];
+          networkConfig = {
+            DNSDefaultRoute = true;
+          };
+          routes = [
+            { Gateway = "10.255.241.1"; }
+            {
+              Destination = "10.255.242.0/24";
+              Gateway = "10.255.241.100";
+            }
+            {
+              Destination = "172.16.239.0/24";
+              Gateway = "10.255.241.210";
+            }
+          ];
+        };
+        networks."40-enp59s0np0" = {
+          DHCP = "no";
+          matchConfig.Name = "enp59s0np0";
+          address = [ "10.255.244.80/24" ];
+        };
+      };
+
+      services.rpcbind.enable = true;
+
+      fileSystems = {
+        "/exports/backup" = {
+          device = "/backup";
+          options = [ "bind" ];
+        };
+        "/exports/ekman" = {
+          device = "/backup/ekman-nfs";
+          options = [ "bind" ];
+        };
+      };
+
+      programs.singularity.enable = true;
+
+      boot.swraid = {
+        enable = true;
+        mdadmConf = ''
+          DEVICE partitions
+          ARRAY /dev/md/0 metadata=1.2 UUID=b743fdd4:5b339cc7:7c43f50f:3b81243e name=fs2:0
+        '';
+      };
+
+      systemd.services.restart-md0 = {
+        description = "restart /dev/md0";
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "sys-devices-virtual-block-md0.device"
+          "-.mount"
+        ];
+        before = [ "backup.mount" ];
+        path = [ "/run/current-system/sw/" ];
+        serviceConfig = {
+          Type = "oneshot";
+        };
+        script = ''
+          restart=0
+          ${util-linux}/bin/lsblk -o MAJ:MIN -n /dev/md0 | grep -q "254:" || restart=1
+          if [ $restart = 1 ]; then
+              ${mdadm}/bin/mdadm --stop /dev/md0
+              ${mdadm}/bin/mdadm --assemble /dev/md0
+              sleep 1
+          fi
+        '';
+      };
+
+      #services.tailscale =  {
+      #  enable = true;
+      #  authKeyFile = "/var/lib/secrets/tailscale.key";
+      #  useRoutingFeatures = "both";
+      #  extraUpFlags = [
+      #    "--login-server=https://headscale.svc.oceanbox.io"
+      #    "--accept-dns=true"
+      #    "--accept-routes=true"
+      #    "--snat-subnet-routes=true"
+      #    "--advertise-routes=10.255.241.0/24"
+      #  ];
+      #};
+
+      #services.networkd-dispatcher = {
+      #  enable = true;
+      #  rules = {
+      #    "tailscale-router" = {
+      #      onState = [ "routable" ];
+      #      script = ''
+      #      #!${pkgs.runtimeShell}
+      #      ${pkgs.ethtool}/bin/ethtool -K eno1 rx-udp-gro-forwarding on
+      #      ${pkgs.ethtool}/bin/ethtool -K eno1 rx-gro-list off
+      #      ${pkgs.ethtool}/bin/ethtool -K eno1 tx-udp-segmentation on
+      #      exit 0
+      #      '';
+      #    };
+      #  };
+      #};
+
+      boot.kernelParams = [
+        "console=tty0"
+        "console=ttyS0,115200"
+      ];
+      systemd.services."serial-getty@ttyS0" = {
+        enable = true;
+        wantedBy = [ "getty.target" ];
+        serviceConfig.Restart = "always";
+      };
+
+      imports = [
+        ./hardware-configuration.nix
+        ../default.nix
+        ../mounts.nix
+      ];
+    };
+}

ekman/fs-backup/hardware-configuration.nix

@@ -0,0 +1,49 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ahci"
+    "xhci_pci"
+    "megaraid_sas"
+    "mpt3sas"
+    "usbhid"
+    "usb_storage"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/19b7e607-b138-442a-9026-3ae1092046c9";
+    fsType = "ext4";
+  };
+
+  #   fileSystems."/backup" =
+  #     { device = "/dev/vg1/data";
+  #       fsType = "xfs";
+  #       options = [ "ro" "noauto" ];
+  #     };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno3.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno4.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode =
+    lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/fs-backup/ssh_host_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCb3/Z4YEFmeeaqQYzATkHgNaDWaT6MjiQsrIFYlZsvdXyE5QjGZMeRBMgJQyoyeCm9ig3sMbY8qiWLc/Kg+rFnEy+4xO7GYzCfqo1YsETiD0K+m/TbOsz9M2m+N8k+REWpKp9njTOYwsjWbrQ6FQPeQaKmig7P0PXJsSS/SAUr0Ns98QtBA9KtfRbDDCUlG4T/+kfu4Y970u0ABOF+zrE2mzfuaSn7n/jJAcAusIUQ9iiTrI6WHAok8lg9jC/+piW6ToEN7/ZX/RDggj5U5ZeN2IsjD1TZq57b/v7BAVx2CA38IYjuYizJFIlL8o7hEP22an1i44crsaR8pwJcfeMWzQxVIm9TPhSZZB9Ibq1R1syNb9vH/svmR8csWl0Uemk3gUGBJwzltmuXhLBg8mobypfBThAI/ZQKslvM9G9r4LoeLmxOOUTbCfXhi3ndIkEC2vGvo74h58BQj/DvMJ3a+V6gTha4837e2LK3Fm/8XWtU42jNbNlmblhLZby7uKWWb6orCnxscRs3cjP3BObSh6h9DO2aXMTnGYSGFdUKrRLFRmzFr+MPdhyORqp+GQgtHkzFG9W3/otQHdDCcD2OeFOxwEr3jOwUgtGIO1ZmJFDB9EIVHWyRk519foeZq/EHcSmIxgbwTgkpi/w8NQ1j++2nDxQOUeWo3U8xiQhONQ== root@fs2

ekman/fs-work/default.nix

@@ -0,0 +1,186 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  name = "fs-work";
+  address = "10.255.241.90";
+  etcdCluster = import ../etcdCluster.nix;
+in
+{
+  fs-work =
+    { config, pkgs, ... }:
+    with pkgs;
+    {
+      deployment.tags = [
+        "fs"
+        "fs-backup"
+      ];
+      deployment.targetHost = address;
+      system.autoUpgrade.enable = lib.mkForce false;
+
+      systemd.targets = {
+        sleep.enable = false;
+        suspend.enable = false;
+        hibernate.enable = false;
+        hybrid-sleep.enable = false;
+      };
+
+      # services.udev.extraRules = ''
+      #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+      # '';
+      environment.systemPackages = with pkgs; [
+        rdma-core
+        hwloc
+        xfsprogs
+      ];
+
+      cluster = {
+        k8sNode = true;
+        slurm = false;
+        mounts = {
+          rdma.enable = true;
+          automount.enable = true;
+          users = true;
+          opt = true;
+          work = false;
+          data = true;
+          ceph = true;
+          backup = false;
+        };
+      };
+
+      features = {
+        host = {
+          inherit address;
+          inherit name;
+        };
+
+        os = {
+          networkmanager.enable = false;
+          externalInterface = "enp33s0f3np3";
+          nfs.enable = true;
+          nfs.exports = ''
+            /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+            /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+            /exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          '';
+        };
+
+        k8s = {
+          enable = true;
+          node.enable = true;
+          master.enable = false;
+          inherit etcdCluster;
+        };
+      };
+
+      systemd.services.rc-local = {
+        description = "rc.local script";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        path = [ "/run/current-system/sw/" ];
+        serviceConfig = {
+          Type = "oneshot";
+        };
+        script = ''
+          # if [ -e /sys/block/md126 ]; then
+          #     echo "deadline" > /sys/block/md126/queue/scheduler
+          #     # echo "4096" > /sys/block/md126/queue/nr_requests
+          #     echo "4096" > /sys/block/md126/queue/read_ahead_kb
+          #     echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
+          #     echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
+          # fi
+          grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
+          grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
+        '';
+      };
+
+      boot.kernel.sysctl = {
+        "vm.dirty_background_ratio" = 5;
+        "vm.dirty_ratio" = 10;
+        "vm.vfs_cache_pressure" = 50;
+        "vm.min_free_kbytes" = 262144;
+      };
+
+      networking = {
+        useNetworkd = true;
+        useDHCP = false;
+        hostName = name;
+        firewall = {
+          allowedTCPPorts = [ ];
+          allowedUDPPorts = [ ];
+          extraCommands = ''
+            # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
+          '';
+        };
+      };
+
+      systemd.network = {
+        networks."40-enp65s0f0np0" = {
+          DHCP = "no";
+          matchConfig.Name = "enp65s0f0np0";
+          address = [ "${address}/24" ];
+          networkConfig = {
+            DNSDefaultRoute = true;
+          };
+          routes = [
+            { Gateway = "10.255.241.1"; }
+            {
+              Destination = "10.255.242.0/24";
+              Gateway = "10.255.241.100";
+            }
+            {
+              Destination = "172.16.239.0/24";
+              Gateway = "10.255.241.210";
+            }
+          ];
+        };
+        networks."40-enp1s0f1np1" = {
+          DHCP = "no";
+          matchConfig.Name = "enp1s0f1np1";
+          address = [ "10.255.244.90/24" ];
+        };
+        networks."42-ibp1s0f0" = {
+          DHCP = "no";
+          matchConfig.Name = "ibp1s0f0 ";
+          address = [ "10.255.243.90/24" ];
+        };
+      };
+
+      services.rpcbind.enable = true;
+
+      fileSystems = {
+        "/exports/work" = {
+          device = "/work";
+          options = [ "bind" ];
+        };
+        "/exports/opt" = {
+          device = "/opt";
+          options = [ "bind" ];
+        };
+      };
+
+      programs.singularity.enable = true;
+
+      security.sudo.extraConfig = ''
+        %sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
+        %admin ALL=(admin) NOPASSWD: ALL
+      '';
+
+      boot.kernelParams = [
+        "console=tty0"
+        "console=ttyS0,115200"
+      ];
+      systemd.services."serial-getty@ttyS0" = {
+        enable = true;
+        wantedBy = [ "getty.target" ];
+        serviceConfig.Restart = "always";
+      };
+
+      imports = [
+        ./hardware-configuration.nix
+        ../default.nix
+        ../mounts.nix
+      ];
+    };
+}

ekman/fs-work/hardware-configuration.nix

@@ -0,0 +1,47 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "smartpqi" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/21d2c2c7-4968-432d-a5c4-08116147b816";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/49B0-2597";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  fileSystems."/work" =
+    { device = "/dev/mapper/data-work";
+      fsType = "xfs";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0f1np1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp65s0f1np1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp6s0f4u1u6.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/fs-work/ssh_host_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxcJ+G4e6TEy0CZ66V6rRvODZ3QBgG5uIPElZr0lAFEe2FLPtiR57puXTV4tFhkY/TsVo0BE5/qDcBUzdRJ0ihuVVruvEUJDr+wDby1+V823xvPYT0IqPNvBBcR/FKIUTWFIhDHCntQ3MWvcp+bXX4ZH02QGmnSNNjyH6krOofuNz0/H8UxpLn/U40ruQx5SyT+uc7t0qryEUAwdvkQ6dGqyeCd4szSyQ1RfaTcd3qqXltxfKx5EbRAxD6pLoiKzYBGzWwKOLczNJGsLoMQ53KhEUyAdEOEx79qxoR4PU8P5q7osZD9NuloHgVT2P1hiNcsNmFzze3WW96PyK9L6Y1NE2rfcrhHIHG8tK31YXBlFQWr/VuPHpvqt21z0r2tPJwM5ui1zgZ1tsEWuKURItqr5Xsxk3D3woY5hT4rtDkV0TLe8/PUTGkGtw15UbQ/BnsFctWe5JnTtZ+90Tjxl4Ty/h0AYilVff6I3tfEln9MBnYig/cOUX0lmo2lnYpUedHYq5LXaga15OW0uonMd8Wl0nSGbqpJ0Gmu+gONJU9AbGchgxxmo0LNbdQ4jANq11LhcK2UZpyPKA7zV0pcgo1WSQyewfdFYIOJzGJPbDhOai/0OAXJYPJa/QJvp08ULpGrUUYXtjmVLw0DCYihm4af84Gr5Fu7hqNO1dplbJh8w== root@nixos

ekman/hive.nix

@@ -0,0 +1,24 @@
+let
+  # Pin the deployment package-set to a specific version of nixpkgs
+  # pkgs = import (builtins.fetchTarball {
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
+  # }) {};
+  pkgs = import <nixpkgs> {};
+
+  ekman-manage = {
+      deployment = {
+          tags = [ "manage" "ekman" ];
+          allowLocalDeployment = true;
+          targetHost = null;
+      };
+      imports = [ ./manage ];
+  };
+
+  login = import ./login { inherit pkgs; };
+  c0 = import ./c0 { inherit pkgs; };
+  c1 = import ./c1 { inherit pkgs; };
+  fs-work = import ./fs-work { inherit pkgs; };
+  fs-backup = import ./fs-backup { inherit pkgs; };
+in
+    { inherit ekman-manage; frontend = ekman-manage; } // login // c0 // c1 // fs-work // fs-backup

ekman/hosts.nix

@@ -0,0 +1,32 @@
+''
+  10.255.241.99 ekman-manage
+  10.255.241.100 ekman-login
+  10.255.241.100 ekman
+  10.255.241.101 c0-1
+  10.255.241.102 c0-2 
+  10.255.241.103 c0-3 
+  10.255.241.104 c0-4 
+  10.255.241.105 c0-5 
+  10.255.241.106 c0-6 
+  10.255.241.107 c0-7 
+  10.255.241.108 c0-8 
+  10.255.241.109 c0-9
+  10.255.241.110 c0-10
+  10.255.241.111 c0-11
+  10.255.241.112 c0-12 
+  10.255.241.113 c0-13 
+  10.255.241.114 c0-14 
+  10.255.241.115 c0-15 
+  10.255.241.116 c0-16
+  10.255.241.117 c0-17 
+  10.255.241.118 c0-18 
+
+  10.255.241.121 c1-1
+  10.255.241.122 c1-2 
+  10.255.241.123 c1-3 
+  10.255.241.124 c1-4 
+  10.255.241.125 c1-5 
+  10.255.241.126 c1-6 
+  10.255.241.127 c1-7 
+  10.255.241.128 c1-8 
+''

ekman/login/default.nix

@@ -0,0 +1,337 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  name = "ekman";
+  address = "10.255.241.100";
+in
+{
+  ekman-login =
+    { config, pkgs, ... }:
+    with pkgs;
+    {
+      deployment.tags = [
+        "login"
+        "cluster"
+      ];
+      deployment.targetHost = address;
+      system.autoUpgrade.enable = lib.mkForce false;
+
+      systemd.targets = {
+        sleep.enable = false;
+        suspend.enable = false;
+        hibernate.enable = false;
+        hybrid-sleep.enable = false;
+      };
+
+      cluster = {
+        compute = true;
+        k8sNode = true;
+        mounts = {
+          rdma.enable = true;
+          automount.enable = true;
+          users = false;
+          opt = false;
+          work = true;
+          data = true;
+          ceph = true;
+          backup = false;
+        };
+      };
+
+      features = {
+        host = {
+          inherit name;
+          inherit address;
+        };
+
+        myvnc.enable = false;
+
+        os = {
+          networkmanager.enable = false;
+          externalInterface = "enp33s0f0np0";
+          nfs.enable = true;
+          nfs.exports = ''
+            /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+            /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          '';
+        };
+
+        hpc = {
+          slurm.server = false;
+          slurm.slurmrestd = false;
+          manageNode = false;
+          loginNode = true;
+          knem = false;
+        };
+
+        k8s = {
+          master.enable = false;
+          node.enable = true;
+        };
+
+        desktop.enable = false;
+        # server.enable = true;
+        monitoring = {
+          #   server = {
+          #     enable = false;
+          #     scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
+          #     defaultAlertReceiver = {
+          #       email_configs = [
+          #         { to = "jonas.juselius@oceanbox.io"; }
+          #       ];
+          #     };
+          #     pageAlertReceiver = {
+          #       webhook_configs = [
+          #         {
+          #           url = "https://prometheus-msteams.k2.itpartner.no/ekman";
+          #           http_config = {
+          #             tls_config = { insecure_skip_verify = true; };
+          #           };
+          #         }
+          #       ];
+          #     };
+          #   };
+          #   webUI.enable = false;
+          #   webUI.acmeEmail = "innovasjon@itpartner.no";
+          #   webUI.allow = [
+          #       "10.1.2.0/24"
+          #       "172.19.254.0/24"
+          #       "172.19.255.0/24"
+          #   ];
+        };
+      };
+
+      # services.udev.extraRules = ''
+      #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+      # '';
+
+      # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_6;
+      services.flannel.iface = "enp33s0f3np3";
+
+      networking = {
+        useNetworkd = true;
+        useDHCP = false;
+        hostName = name;
+        firewall = {
+          allowedTCPPorts = [ 6443 ];
+          extraCommands = ''
+            # needed for nodeport access on k1 and k2
+            # iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.2
+            iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
+          '';
+        };
+      };
+
+      systemd.network = {
+        networks = {
+          "40-enp33s0f0np0" = {
+            DHCP = "no";
+            matchConfig.Name = "enp33s0f0np0";
+            address = [ "10.255.242.2/24" ];
+            routes = [
+              { Gateway = "10.255.242.1"; }
+            ];
+          };
+          "40-enp33s0f3np3" = {
+            DHCP = "no";
+            matchConfig.Name = "enp33s0f3np3";
+            address = [ "${address}/24" ];
+            networkConfig = {
+              DNSDefaultRoute = true;
+            };
+            routes = [
+              {
+                Destination = "172.16.239.0/24";
+                Gateway = "10.255.241.210";
+              }
+            ];
+          };
+          "41-enp65s0f1np1" = {
+            DHCP = "no";
+            matchConfig.Name = "enp65s0f1np1";
+            address = [ "10.255.244.100/24" ];
+          };
+          "45-ibp65s0f0" = {
+            DHCP = "no";
+            matchConfig.Name = "ibp65s0f0";
+            address = [ "10.255.243.100/24" ];
+          };
+        };
+      };
+
+      services.resolved = {
+        # DNS=[::1]:53
+        extraConfig = ''
+          DNSStubListener=no
+        '';
+      };
+
+      fileSystems = {
+        "/exports/users" = {
+          device = "/home";
+          options = [ "bind" ];
+        };
+        "/exports/opt/bin" = {
+          device = "/opt/bin";
+          options = [ "bind" ];
+        };
+        "/exports/opt/sif" = {
+          device = "/opt/sif";
+          options = [ "bind" ];
+        };
+        "/exports/nfs-provisioner" = {
+          device = "/vol/nfs-provisioner";
+          options = [ "bind" ];
+        };
+        "/users" = {
+          device = "/home";
+          options = [ "bind" ];
+        };
+        "/vol/local-storage/vol1" = {
+          device = "/vol/vol1";
+          options = [ "bind" ];
+        };
+        "/vol/local-storage/vol2" = {
+          device = "/vol/vol2";
+          options = [ "bind" ];
+        };
+      };
+
+      nix.extraOptions = ''
+        secret-key-files = /etc/nix/ekman.key
+      '';
+
+      # services.xserver = {
+      #   enable = false;
+      #   enableCtrlAltBackspace = true;
+      #   layout = "us";
+      #   xkbVariant = "altgr-intl";
+      #   xkbOptions = "eurosign:e";
+      #   displayManager = {
+      #     gdm.enable = false;
+      #     job.logToFile = true;
+      #   };
+      #   # desktopManager.xfce.enable = true;
+      # };
+
+      services.prometheus.alertmanager.configuration.global = {
+        smtp_smarthost = "smtpgw.itpartner.no";
+        # smtp_auth_username = "utvikling";
+        # smtp_auth_password = "S0m3rp0m@de#21!";
+        smtp_hello = "ekman.oceanbox.io";
+        smtp_from = "noreply@ekman.oceanbox.io";
+      };
+
+      # services.nginx = {
+      #   virtualHosts = {
+      #     "ds.matnoc.regnekraft.io" = {
+      #       forceSSL = true;
+      #       enableACME = true;
+      #       serverAliases = [];
+      #       locations."/" = {
+      #         proxyPass = "http://localhost:9088";
+      #         proxyWebsockets = false;
+      #         extraConfig = ''
+      #           allow 10.1.2.0/24;
+      #           allow 172.19.254.0/24;
+      #           allow 172.19.255.0/24;
+      #           deny all;
+      #         '';
+      #       };
+      #     };
+      #   };
+      # };
+
+      # services.gitlab-runner = {
+      #   enable = true;
+      #   extraPackages = with pkgs; [
+      #     singularity
+      #   ];
+      #   concurrent = 4;
+      #   services = {
+      #     sif = {
+      #       registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
+      #       executor = "shell";
+      #       tagList = [ "ekman" "sif" ];
+      #     };
+      #   };
+      # };
+
+      security.sudo.extraConfig = ''
+        %sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
+        %admin ALL=(admin) NOPASSWD: ALL
+        # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
+      '';
+
+      security.pam = {
+        services.sshd.googleAuthenticator.enable = true;
+        loginLimits = [
+          {
+            domain = "@users";
+            item = "rss";
+            type = "hard";
+            value = 16000000;
+          }
+          {
+            domain = "@users";
+            item = "cpu";
+            type = "hard";
+            value = 180;
+          }
+        ];
+      };
+
+      system.activationScripts = {
+        home-permissions.text = ''
+          chmod 755 /home/olean
+          chmod 755 /home/frankgaa
+          chmod 755 /home/jonas
+          chmod 755 /home/mrtz
+          chmod 755 /home/avle
+          chmod 755 /home/stig
+          chmod 755 /home/bast
+          chmod 755 /home/simenlk
+          chmod 755 /work/kraken
+        '';
+      };
+
+      # ssh-rsa is deprecated, but putty/winscp users use it
+      services.openssh.extraConfig = ''
+        # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
+        PubkeyAuthOptions verify-required
+      '';
+
+      environment.systemPackages = [ ];
+
+      virtualisation.docker.enable = pkgs.lib.mkForce true;
+
+      services.tailscale = {
+        enable = true;
+        authKeyFile = "/var/lib/secrets/tailscale.key";
+        useRoutingFeatures = "client";
+        extraUpFlags = [
+          "--login-server=https://headscale.svc.oceanbox.io"
+          "--accept-dns=true"
+          "--advertise-tags=tag:hpc"
+        ];
+      };
+
+      boot.kernelParams = [
+        "console=tty0"
+        "console=ttyS0,115200"
+      ];
+      systemd.services."serial-getty@ttyS0" = {
+        enable = true;
+        wantedBy = [ "getty.target" ];
+        serviceConfig.Restart = "always";
+      };
+
+      imports = [
+        ./hardware-configuration.nix
+        ../default.nix
+        ../mounts.nix
+        ../myvnc.nix
+      ];
+    };
+}

ekman/login/hardware-configuration.nix

@@ -0,0 +1,39 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/c4a5ea80-2a87-440d-8431-a695d623da24";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/D8A3-A04D";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

ekman/login/ssh_host_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDSfhEB+DPHCWHmdzf+Xea7N9A/Pd98Nh9ChcQqVcjJNUMNeOta3NIdW5m9sDhWlB6kioMEJa4DbdAfJsKciGnSh9VJeerfnPFgov3WIr5RfoTAfKRE5GKW28c1f4m0WpfZkh5gvS0mAT7jkxwvTcQI9PViteIb4muAlbLPCqDUhFWC9yzSHzSt9MSMjpPY18GlsWyPM2ctT3OUq28wWlcleu7pFEFIpLKj+tqpBDBvS1I0fKagR0eDmCUGGbT8Fth+Jodt66k+QmJ28A58DfiDPECeN1/AzhwOMRYYMLSFmrLRmy/gML1egiZM+zwDKTRQ9lHhFjO8SOfq0F/A4NxFXegmXI1QbbtsG8KrLEDd9a8pZG99YdRa1dE5TA3H5iAnekN7rO0vfnlJfGMQOdI2I9JtqX48k98wUjxmv0EiUJV68j0rQaEwDQFjbEmiN2nEA9sm6+CMKCf3PywvDAijzAzSicA5Pm9MfNQLiXTXZbB7biJlNLfSlSOZrOP7Rd1VZ+bF9lYou88Y3/BZ/S23NdMVr9DEVYu2pUOCEgeXwVW32LHEgDfUslp7ZDUICGgLMQX5Wpz8f+zByRUmE2ovhCjcT71TVn/paIqjjy+2sgDX+BVSULQQAoPS56by1jfMIZjjNqDWE86yY3uWkeobmY5C55s8q2Hlyv+8meq6LQ== root@nixos

ekman/manage/default.nix

@@ -0,0 +1,408 @@
+{ pkgs, ... }:
+let
+  computeNodes =
+    import ../c0/nodes.nix
+    ++ import ../c1/nodes.nix
+    ++ [
+      rec {
+        idx = 100;
+        name = "ekman";
+        address = "10.255.241.${toString idx}";
+        ipoib = "10.255.243.${toString idx}";
+        pubkey = ../login/ekman.pub;
+      }
+      rec {
+        idx = 90;
+        name = "fs-work";
+        address = "10.255.241.${toString idx}";
+        ipoib = "10.255.243.${toString idx}";
+        pubkey = ../fs-work/fs-work.pub;
+      }
+      rec {
+        idx = 81;
+        name = "fs-backup";
+        address = "10.255.241.${toString idx}";
+        ipoib = "10.255.243.${toString idx}";
+        pubkey = ../fs-backup/fs-backup.pub;
+      }
+    ];
+  etcdCluster = import ../etcdCluster.nix;
+  name = "ekman-manage";
+  address = "10.255.241.99";
+  ipoib = "10.255.243.99";
+in
+{
+  systemd.targets = {
+    sleep.enable = false;
+    suspend.enable = false;
+    hibernate.enable = false;
+    hybrid-sleep.enable = false;
+  };
+
+  # services.udev.extraRules = ''
+  #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+  # '';
+
+  environment.systemPackages = with pkgs; [
+    rdma-core
+    hwloc
+    headscale
+  ];
+
+  cluster = {
+    k8sNode = true;
+    compute = false;
+    slurm = true;
+    mounts = {
+      rdma.enable = true;
+      automount.enable = true;
+      users = false;
+      opt = true;
+      work = false;
+      data = true;
+      ceph = true;
+      backup = false;
+    };
+  };
+
+  features = {
+    desktop.enable = false;
+    cachix.enable = false;
+
+    host = {
+      inherit address;
+      inherit name;
+    };
+
+    myvnc.enable = false;
+
+    os = {
+      externalInterface = "eno1";
+      networkmanager.enable = false;
+      nfs.enable = false;
+      nfs.exports = ''
+        /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+        /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+      '';
+    };
+
+    hpc = {
+      slurm.server = true;
+      slurm.slurmrestd = true;
+      slurm.dbdServer = false;
+      manageNode = true;
+    };
+
+    k8s = {
+      master.enable = true;
+      node.enable = true;
+      nodes = computeNodes;
+      inherit etcdCluster;
+    };
+
+    monitoring = {
+      server = {
+        enable = false;
+        scrapeHosts = [
+          "ekman-manage"
+          "ekman"
+          "fs-work"
+          "fs-backup"
+        ]
+        ++ (builtins.map (x: x.name) computeNodes);
+        defaultAlertReceiver = {
+          email_configs = [
+            { to = "jonas.juselius@oceanbox.io"; }
+          ];
+        };
+        pageAlertReceiver = {
+          webhook_configs = [
+            {
+              url = "https://prometheus-msteams.k2.itpartner.no/ekman";
+              http_config = {
+                tls_config = {
+                  insecure_skip_verify = true;
+                };
+              };
+            }
+          ];
+        };
+      };
+      webUI.enable = false;
+      webUI.acmeEmail = "acme@oceanbox.io";
+      webUI.allow = [
+        "10.1.2.0/24"
+        "172.19.254.0/24"
+        "172.19.255.0/24"
+      ];
+      infiniband-exporter = {
+        enable = true;
+        nameMap = ''
+               0xe8ebd3030024a2c6 "ekman"
+               0x0c42a10300ddc4bc "ekman-manage"
+               0xe8ebd3030024a2ae "fs-work"
+               0x1c34da0300787798 "fs-backup"
+               0xe8ebd3030024981e "c0-1"
+               0xe8ebd3030024a21a "c0-2"
+          	    0xe8ebd30300249a3a "c0-3"
+               0xe8ebd30300248b9e "c0-4"
+               0xe8ebd30300248b86 "c0-5"
+               0xe8ebd3030024998a "c0-6"
+               0xe8ebd30300248b8e "c0-7"
+               0xe8ebd3030024999e "c0-8"
+               0xe8ebd30300248fca "c0-9"
+               0xe8ebd3030024a216 "c0-10"
+               0xe8ebd30300248b96 "c0-11"
+               0xe8ebd30300248b9a "c0-12"
+               0xe8ebd303002495d2 "c0-13"
+               0xe8ebd303002495e2 "c0-14"
+               0xe8ebd30300248f42 "c0-15"
+               0xe8ebd303002495e6 "c0-16"
+               0x0c42a10300dbe7f4 "c1-1"
+               0x0c42a10300dbe7d8 "c1-2"
+               0x0c42a10300dbe800 "c1-3"
+               0x0c42a10300dbec80 "c1-4"
+               0x0c42a10300dbea50 "c1-5"
+               0x0c42a10300dbeb2c "c1-6"
+               0x0c42a10300dbe7fc "c1-7"
+               0x0c42a10300dbe5a0 "c1-8"
+        '';
+      };
+      slurm-exporter = {
+        enable = true;
+        port = 6080;
+      };
+    };
+  };
+
+  programs.singularity.enable = true;
+
+  # services.udev.extraRules = ''
+  #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
+  # '';
+
+  services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
+
+  services.flannel.iface = "eno2";
+
+  networking = {
+    useNetworkd = true;
+    hostName = name;
+    firewall = {
+      checkReversePath = "loose";
+      trustedInterfaces = [ "tailscale0" ];
+      allowedUDPPorts = [
+        41641
+        53
+      ];
+      allowedTCPPorts = [
+        6443
+        4725
+        53
+      ];
+      extraCommands = ''
+        # needed for nodeport access on k1 and k2
+        # iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
+        iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
+        # iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
+        # iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE
+        # iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
+        # iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -j SNAT --to-source 10.255.242.3
+      '';
+    };
+  };
+
+  systemd.network = {
+    networks = {
+      eno1 = {
+        DHCP = "no";
+        matchConfig.Name = "eno1";
+        address = [ "10.255.242.3/24" ];
+        routes = [ { Gateway = "10.255.242.1"; } ];
+      };
+      eno2 = {
+        DHCP = "no";
+        matchConfig.Name = "eno2";
+        address = [ "${address}/24" ];
+        routes = [
+          {
+            Destination = "172.16.239.0/24";
+            Gateway = "10.255.241.210";
+          }
+        ];
+      };
+      ens2f1np1 = {
+        DHCP = "no";
+        matchConfig.Name = "ens2f1np1";
+        address = [ "10.255.244.99/24" ];
+      };
+      ibs2f0 = {
+        DHCP = "no";
+        matchConfig.Name = "ibs2f0";
+        address = [ "${ipoib}/24" ];
+      };
+    };
+  };
+
+  services.resolved = {
+    # DNS=[::1]:53
+    extraConfig = ''
+      DNSStubListener=no
+    '';
+  };
+
+  fileSystems = {
+    "/exports/public" = {
+      device = "/srv/public";
+      options = [ "bind" ];
+    };
+  };
+
+  nix.extraOptions = ''
+    # secret-key-files = /etc/nix/ekman.key
+  '';
+
+  services.prometheus.alertmanager.configuration.global = {
+    smtp_smart_host = "oceanbox-io.mail.protection.outlook.com";
+    smtp_tls_config = {
+      ca_file = "/etc/ssl/certs/ca-bundle.crt";
+      cert_file = "/var/lib/secrets/ekman-lb.oceanbox.io.pem";
+      key_file = "/var/lib/secrets/ekman-lb.oceanbox.io-key.pem";
+    };
+    smtp_hello = "ekman.oceanbox.io";
+    smtp_from = "noreply@oceanbox.io";
+  };
+
+  security.pam = {
+    services.sshd.googleAuthenticator.enable = true;
+    loginLimits = [
+      {
+        domain = "@users";
+        item = "rss";
+        type = "hard";
+        value = 16000000;
+      }
+      {
+        domain = "@users";
+        item = "cpu";
+        type = "hard";
+        value = 180;
+      }
+    ];
+  };
+
+  system.activationScripts = {
+    home-permissions.text = ''
+      chmod 755 /home/olean
+      chmod 755 /home/frankgaa
+      chmod 755 /home/jonas
+      chmod 755 /home/stig
+      chmod 755 /home/bast
+      chmod 755 /home/mrtz
+      chmod 755 /home/avle
+      chmod 755 /home/simenlk
+      chmod 755 /home/ole
+    '';
+  };
+
+  # ssh-rsa is deprecated, but putty/winscp users use it
+  services.openssh.extraConfig = ''
+    # pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
+    PubkeyAuthOptions verify-required
+  '';
+
+  # boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
+
+  virtualisation.docker.enable = pkgs.lib.mkForce true;
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "acme@oceanbox.io";
+    defaults.dnsPropagationCheck = false;
+    certs."ekman-lb.oceanbox.io" = {
+      dnsProvider = "namecheap";
+      environmentFile = "/var/lib/secrets/namecheap.key";
+      dnsPropagationCheck = false;
+      postRun = ''
+        cp fullchain.pem /var/lib/secrets/ekman-lb.oceanbox.io.pem
+        cp key.pem /var/lib/secrets/ekman-lb.oceanbox.io-key.pem
+        chmod 644 /var/lib/secrets/ekman-lb.oceanbox.io.pem
+        chmod 640 /var/lib/secrets/ekman-lb.oceanbox.io-key.pem
+      '';
+    };
+  };
+
+  programs.msmtp = {
+    enable = true;
+    accounts = {
+      default = {
+        auth = false;
+        host = "oceanbox-io.mail.protection.outlook.com";
+        port = 25;
+        from = "noreply@ekman.oceanbox.io";
+        tls_starttls = true;
+        tls = true;
+        tls_trust_file = "/etc/ssl/certs/ca-bundle.crt";
+        tls_key_file = "/var/lib/secrets/ekman-lb.oceanbox.io-key.pem";
+        tls_cert_file = "/var/lib/secrets/ekman-lb.oceanbox.io.pem";
+      };
+    };
+    defaults = {
+      aliases = "/etc/aliases";
+    };
+  };
+
+  environment.etc = {
+    "aliases" = {
+      text = ''
+        root: jonas.juselius@oceanbox.io, moritz.jorg@oceanbox.io
+      '';
+      mode = "0644";
+    };
+  };
+
+  services.tailscale = {
+    enable = true;
+    authKeyFile = "/var/lib/secrets/tailscale.key";
+    useRoutingFeatures = "both"; # for exit-node usage
+    extraUpFlags = [
+      "--login-server=https://headscale.svc.oceanbox.io"
+      "--accept-dns=true"
+      "--accept-routes=false"
+      "--advertise-tags=tag:hpc"
+    ];
+  };
+  services.networkd-dispatcher = {
+    enable = true;
+    rules = {
+      "tailscale-router" = {
+        onState = [ "routable" ];
+        script = ''
+          #!${pkgs.runtimeShell}
+          ${pkgs.ethtool}/bin/ethtool -K eno1 rx-udp-gro-forwarding on
+          ${pkgs.ethtool}/bin/ethtool -K eno1 rx-gro-list off
+          ${pkgs.ethtool}/bin/ethtool -K eno1 tx-udp-segmentation on
+          exit 0
+        '';
+      };
+    };
+  };
+
+  boot.kernelParams = [
+    "console=tty0"
+    "console=ttyS0,115200"
+  ];
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+
+  imports = [
+    ./hardware-configuration.nix
+    ../default.nix
+    ../mounts.nix
+    ../myvnc.nix
+    ../../dns.nix
+  ];
+}

ekman/manage/hardware-configuration.nix

@@ -0,0 +1,27 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/099be829-726d-44c9-b113-4c2604a663c5";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/8111-9F24";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+}

ekman/manage/ssh_host_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkSSuHkieXwgFMKRy4MjwjNrJEItWbQHeAAH+Zn1YZp root@localhost

ekman/mounts.nix

@@ -0,0 +1,150 @@
+{ lib, config, ... }:
+with lib;
+let
+  cfg = config.cluster.mounts;
+
+  subnet =
+    if cfg.rdma.enable then "243"
+    else if cfg.gbe100.enable then "244"
+    else "241";
+
+  options =
+    [ "soft" "defaults" "vers=4.2" ] ++
+    (if cfg.automount.enable then [ "x-systemd.automount" ] else []);
+
+  users =
+    if cfg.users then {
+      "/users" = {
+        device = "10.255.241.100:/users";
+        fsType = "nfs4";
+        options = [
+          "soft"
+          "defaults"
+          "x-systemd.automount"
+        ];
+      };
+    } else {};
+
+  opt =
+    let
+      server = "10.255.241.100";
+    in
+    if cfg.opt then {
+      "/opt/bin" = {
+        device = "${server}:/opt/bin";
+        fsType = "nfs4";
+        inherit options;
+      };
+      "/opt/sif" = {
+        device = "${server}:/opt/sif";
+        fsType = "nfs4";
+        inherit options;
+      };
+    } else {};
+
+  data =
+      if cfg.ceph then {
+        "/data" = {
+          device = "/ceph";
+          options = [ "bind" ];
+        };
+      } else if cfg.backup then {
+         "/data" = {
+          device = "/backup";
+          options = [ "bind" ];
+        };
+      } else {};
+      #     device = "10.255.${subnet}.80:/backup";
+      #     # device = "10.255.${subnet}.80:/data";
+      #     fsType = "nfs4";
+      #     inherit options;
+      #   };
+
+  work =
+      if cfg.work then {
+        "/work" = {
+          device = "10.255.${subnet}.90:/work";
+          fsType = "nfs4";
+          options =  options ++ (if cfg.rdma.enable then [ "rdma" ] else []);
+        };
+    } else {};
+
+  backup =
+      if cfg.backup then {
+        "/backup" = {
+          device = "10.255.${subnet}.80:/backup";
+          fsType = "nfs4";
+          options = options ++ [ "ro" ] ++ (if cfg.rdma.enable then [ "rdma" ] else []);
+        };
+    } else {};
+
+  ceph  =
+    if cfg.ceph then {
+        "/ceph" = {
+          device = "oceanbox@.data=/";
+          fsType = "ceph";
+          options = [
+            "mon_addr=10.255.241.30/10.255.241.31/10.255.241.32:6789"
+            "_netdev"
+          ];
+        };
+      } else {};
+
+  fileSystems = users // opt // data // work // backup // ceph;
+
+  automount = mountpoint:
+      if cfg.automount.enable && builtins.hasAttr mountpoint fileSystems then
+        [{
+          wantedBy = [ "multi-user.target" ];
+          automountConfig = {
+            TimeoutIdleSec = "600";
+          };
+          where = mountpoint;
+        }]
+      else [];
+
+  automounts =
+      [] ++
+      automount "/work" ++
+      automount "/opt" ++
+      automount "/backup" ++
+      automount "/data";
+
+  cephConf =
+   if cfg.ceph then {
+      "ceph/ceph.conf" = {
+        text = ''
+          [global]
+          mon_host = 10.255.241.30:6789,10.255.241.31:6789,10.244.241.32:6789
+          log file = /tmp/ceph-$pid.log
+          [client.oceanbox]
+          key = AQDQNgRm6IE7JxAA1glJKsWPIBB/H/GxFYM0vQ==
+          [client.rbd]
+          key = AQCjth9mjR41ABAAvSs6hltidQT6Hu5OKwWu+Q==
+        '';
+        mode = "0660";
+        group = "admin";
+      };
+    } else {};
+
+in
+{
+  options.cluster.mounts = {
+    rdma.enable = mkEnableOption "Enable NFS over RDMA";
+    gbe100.enable = mkEnableOption "Enable NFS over 100 GbE";
+    automount.enable = mkEnableOption "Enable NFS automounting";
+    users = mkEnableOption "Enable /users";
+    opt = mkEnableOption "Enable /opt";
+    data = mkEnableOption "Enable /data";
+    work = mkEnableOption "Enable /work";
+    backup = mkEnableOption "Enable /backup";
+    ceph = mkEnableOption "Enable /ceph";
+  };
+
+  config = {
+    fileSystems = fileSystems;
+    environment.etc = cephConf;
+    systemd.automounts = automounts;
+    boot.kernelModules = [ "rbd" ];
+  };
+}

ekman/myvnc.nix

@@ -0,0 +1,44 @@
+{ pkgs, lib, config, ... }:
+with lib;
+let
+  cfg = config.features.myvnc;
+
+  myvnc = pkgs.writeScriptBin "myvnc" ''
+    #!${pkgs.runtimeShell}
+
+    uid=`id -u`
+    port=$((9000+$uid))
+    shell=`getent passwd $(id -un) | awk -F : '{print $NF}'`
+    vnc=${pkgs.turbovnc}/bin/vncserver
+    # vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver
+
+    case $1 in
+    -p|--port) shift; port=$1 ;;
+    kill|stop)
+        display=$($vnc -list | sed  -n 's/^\(:[0-9]\+\).*/\1/p'| head -1)
+        $vnc -kill $display
+        exit 0
+        ;;
+    esac
+    ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1
+    [ $? = 1 ] && $vnc -rfbport $port
+    echo "Xvnc server is running on port $port."
+    # exec $shell -i
+  '';
+
+  configuration = {
+      services.xserver.windowManager.fluxbox.enable = true;
+      environment.systemPackages = with pkgs; [
+          alacritty
+          myvnc
+      ];
+  };
+in {
+  options.features.myvnc = {
+    enable = mkEnableOption "Enable mynvc script";
+  };
+
+  config = mkMerge [
+    (mkIf config.features.myvnc.enable configuration)
+  ];
+}

ekman/overlays.nix

@@ -0,0 +1,31 @@
+self: super:
+let
+  msmtp = super.msmtp.overrideAttrs (attrs: rec {
+    configureFlags = attrs.configureFlags ++ [ "--with-tls=openssl" ];
+    buildInputs = attrs.buildInputs ++ [ super.openssl ];
+  });
+
+  lib = super.lib;
+  squashfsTools = super.squashfsTools;
+  cryptsetup = super.cryptsetup;
+
+  singularity = super.singularity.overrideAttrs (attrs: rec {
+      installPhase = ''
+        runHook preInstall
+        make -C builddir install LOCALSTATEDIR=$out/var
+        chmod 755 $out/libexec/singularity/bin/starter-suid
+
+        # Explicitly configure paths in the config file
+        sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf
+        sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf
+        sed -i 's|^shared loop devices = no|shared loop devices = yes|' $out/etc/singularity/singularity.conf
+
+        runHook postInstall
+      '';
+  });
+in
+{
+    inherit singularity;
+    # inherit msmtp;
+}
+

gw/default.nix

@@ -0,0 +1,141 @@
+{ pkgs, ... }:
+{
+  system.autoUpgrade.enable = pkgs.lib.mkForce false;
+
+  networking = {
+    useNetworkd = true;
+    nameservers = [ "127.0.0.1" ];
+    domain = "oceanbox.io";
+    search = [
+      "obx"
+      "oceanbox.io"
+    ];
+    useDHCP = false;
+    extraHosts = import ../hosts.nix + import ./hosts.nix;
+  };
+
+  environment.systemPackages = with pkgs; [
+    iperf
+    zellij
+    fping
+  ];
+
+  services.resolved = {
+    enable = true;
+    dnssec = "false";
+    extraConfig = ''
+      DNSStubListener=no
+    '';
+    fallbackDns = [
+      "1.1.1.1"
+      "1.0.0.1"
+    ];
+  };
+
+  boot = {
+    loader.systemd-boot.enable = true;
+    loader.efi.canTouchEfiVariables = true;
+  };
+
+  console = {
+    font = "Lat2-Terminus16";
+    keyMap = "us";
+  };
+
+  i18n = {
+    defaultLocale = "en_US.UTF-8";
+    extraLocaleSettings = {
+      LC_CTYPE = "en_DK.UTF-8";
+      LC_TIME = "en_DK.UTF-8";
+      LC_PAPER = "en_DK.UTF-8";
+      LC_NAME = "en_DK.UTF-8";
+      LC_ADDRESS = "en_DK.UTF-8";
+      LC_TELEPHONE = "en_DK.UTF-8";
+      LC_MEASUREMENT = "en_DK.UTF-8";
+      LC_IDENTIFICATION = "en_DK.UTF-8";
+    };
+  };
+
+  time.timeZone = "Europe/Oslo";
+
+  features = {
+    desktop.enable = false;
+    laptop.enable = false;
+    desktop.wayland.enable = false;
+    desktop.hyprland.enable = false;
+    cachix.enable = false;
+
+    # host = {
+    #   inherit name;
+    #   inherit address;
+    # };
+
+    pki = {
+      enable = false;
+      certmgr.enable = true;
+      certs = {
+        foo = {
+          hosts = [ "localhost" ];
+        };
+      };
+    };
+
+    os = {
+      networkmanager.enable = false;
+      docker.enable = true;
+
+      adminAuthorizedKeys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas-3"
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
+        "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAlHx2RWPvBhYzevQE+llnyDuInSsyhs6PFaoavtEB2VLr2gOFxDZW5VmDXlorAXtCpcPpJdYbnPuRWZd0m5950BatWy5UPgDIK2Qm5XABlq6/tXzHJme9PJHcXwo6WStaB31RiJs9nhdYIhPRjcbeFdx/7Fz3atVikE6YUtb8a7YGeKgZh6ashtYPpG3oSBLn1menjk6CxVRt16de3PoDlYav/J8WPbaGJqcPIU9rjzqJkI4aG1txSyJ8Vt2vawQKnzPZCuQAdWgquE3CbJkJbgoh2TKmHds71WMTg3ZXz2KfVjdN8IXCTxdySlgVVAN6x0usGVnA22XnMYQEgbcb3Q== rsa-key-20201215"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII77Aa2MFZMTha8PdkNg32UR8y6Hwb4R0aR9Ad9qifNq mrtz@wurst"
+      ];
+    };
+  };
+
+  programs.singularity.enable = false;
+
+  nixpkgs.config.allowUnfreee = true;
+
+  # Wait for resolved and network-online
+  systemd.services.tailscaled.after = [
+    "network-online.target"
+    "systemd-resolved.service"
+  ];
+
+  # Don't wait on tailscale
+  systemd.network.wait-online.ignoredInterfaces = [ "tailscale0" ];
+
+  services.networkd-dispatcher = {
+    enable = true;
+    rules = {
+      "tailscale-router" = {
+        onState = [ "routable" ];
+        script = ''
+          #!${pkgs.runtimeShell}
+          ${pkgs.ethtool}/bin/ethtool -K ens3 rx-udp-gro-forwarding on
+          ${pkgs.ethtool}/bin/ethtool -K ens3 rx-gro-list off
+          ${pkgs.ethtool}/bin/ethtool -K ens3 tx-udp-segmentation on
+          exit 0
+        '';
+      };
+    };
+  };
+
+  system.activationScripts.system-diff = {
+    supportsDryActivation = true; # safe: only outputs to stdout
+    text = ''
+      export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
+      if [ -e /run/current-system ]; then
+        ${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
+      fi
+    '';
+  };
+
+  imports = [
+    ./users.nix
+    ../dns.nix
+    ../modules
+    ../nixos
+  ];
+}

gw/hel1-1/default.nix

@@ -0,0 +1,102 @@
+{ pkgs }:
+let
+  name = "gw-hel1-1";
+  address = "37.27.203.38";
+in
+{
+  hel1-1 =
+    { config, pkgs, ... }:
+    with pkgs;
+    {
+      deployment.tags = [ "gw" ];
+      deployment.targetHost = "100.64.0.34";
+
+      services.openssh.openFirewall = false;
+
+      networking = {
+        useNetworkd = true;
+        useDHCP = false;
+        hostName = name;
+        firewall = {
+          interfaces.enp1s0 = {
+            allowedTCPPorts = [ ];
+            allowedUDPPorts = [ ];
+          };
+          interfaces.enp7s0 = {
+            allowedTCPPorts = [
+              111
+              2049
+              53
+              22
+              6819
+            ];
+            allowedUDPPorts = [
+              111
+              2049
+              24007
+              24008
+              53
+              22
+            ];
+          };
+          extraCommands = ''
+            iptables -t nat -A POSTROUTING -s 10.0.0.0/16 -o enp1s0 -j MASQUERADE
+          '';
+        };
+      };
+
+      systemd.network = {
+        # wait-online.enable = false;
+        networks."40-enp1s0" = {
+          matchConfig.Name = "enp1s0";
+          DHCP = "yes";
+        };
+        networks."40-enp7s0" = {
+          matchConfig.Name = "enp7s0";
+          DHCP = "yes";
+          # address = [ "10.0.1.1/24" ];
+        };
+      };
+
+      programs.ssh.knownHosts = {
+        hel1-gw-1 = {
+          hostNames = [
+            name
+            "${name}.hel1.obx"
+            address
+          ];
+          publicKeyFile = ./ssh_host_key.pub;
+        };
+      };
+
+      features = {
+        os = {
+          externalInterface = "enp1s0";
+          nfs.enable = true;
+          nfs.openFirewall = false;
+          nfs.exports = ''
+            /vol/export 10.0.1.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          '';
+        };
+      };
+
+      services.tailscale = {
+        enable = true;
+        authKeyFile = "/var/lib/secrets/tailscale.key";
+        useRoutingFeatures = "both";
+        extraUpFlags = [
+          "--login-server=https://headscale.svc.oceanbox.io"
+          "--accept-dns=true"
+          "--advertise-routes=10.0.1.0/24"
+          "--advertise-routes=tag:hel1"
+        ];
+      };
+
+      imports = [
+        ./hardware-configuration.nix
+        ../default.nix
+        ../../users.nix
+        ../../dns.nix
+      ];
+    };
+}

gw/hel1-1/hardware-configuration.nix

@@ -0,0 +1,54 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "ahci"
+    "xhci_pci"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/01e1966b-8621-4a99-9260-a4a94f9d1641";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/4EF7-97DA";
+    fsType = "vfat";
+    options = [
+      "fmask=0022"
+      "dmask=0022"
+    ];
+  };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp7s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}