platform/rossby/fs-work/default.nix

{ pkgs ? import <nixpkgs> {} }:
let
  # Pin the deployment package-set to a specific version of nixpkgs
  # pkgs = import (builtins.fetchTarball {
  #   url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
  #   sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
  # }) {};
  # pkgs = import <nixpkgs> {};
  etcdCluster = import ../etcdCluster.nix;
  name = "fs-work";
  address = "172.16.239.210";
in {
  fs-work = { config, pkgs, ... }: with pkgs; {
    deployment.tags = [ "fs" "fs-work" "cluster" ];
    deployment.targetHost = address;
    system.autoUpgrade.enable = lib.mkForce false;

    systemd.targets = {
      sleep.enable = false;
      suspend.enable = false;
      hibernate.enable = false;
      hybrid-sleep.enable = false;
    };

    # services.udev.extraRules = ''
    #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
    # '';
    security.sudo.extraConfig = ''
      %sif ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
      %admin ALL=(admin) NOPASSWD: ALL
    '';

    environment.systemPackages = with pkgs; [
        rdma-core
        hwloc
        xfsprogs
    ];

    cluster = {
      k8sNode = true;
      slurm = false;
      mounts = {
        rdma.enable = false;
        automount.enable = true;
        users = false;
        opt = false;
        work = false;
        data = false;
        ceph = false;
      };
    };

    features = {
      host = {
        inherit address;
        inherit name;
      };

      os = {
        networkmanager.enable = false;
        externalInterface = "enp1s0np0";
        nfs.enable = true;
        nfs.exports = ''
          /exports 172.16.239.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
        '';
      };

      k8s = {
          enable = true;
          node.enable = true;
          master.enable = false;
          inherit etcdCluster;
      };
    };

    systemd.services.rc-local = {
        description = "rc.local script";
        wantedBy = [ "multi-user.target" ];
        after = [ "network.target" ];
        path = [ "/run/current-system/sw/" ];
        serviceConfig = {
          Type = "oneshot";
        };
        script = ''
            # if [ -e /sys/block/md126 ]; then
            #     echo "deadline" > /sys/block/md126/queue/scheduler
            #     # echo "4096" > /sys/block/md126/queue/nr_requests
            #     echo "4096" > /sys/block/md126/queue/read_ahead_kb
            #     echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
            #     echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
            # fi
            grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
            grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
        '';
    };

    boot.kernel.sysctl = {
        "vm.dirty_background_ratio" = 5;
        "vm.dirty_ratio" = 10;
        "vm.vfs_cache_pressure" = 50;
        "vm.min_free_kbytes" = 262144;
    };

    networking = {
      useNetworkd = true;
      useDHCP = false;
      hostName = name;
      firewall = {
        allowedTCPPorts = [];
        allowedUDPPorts = [];
        extraCommands = ''
          # iptables -t nat -A POSTROUTING -s 172.16.239.0/24 -j MASQUERADE
        '';
      };
    };

    systemd.network = {
      networks."40-enp1s0np0" = {
        matchConfig.Name = "enp1s0np0";
        DHCP = "no";
        address = [ "${address}/24" ];
        routes = [
          { Gateway = "172.16.239.1"; }
          {
            Destination = "10.255.241.0/24";
            Gateway = "172.16.239.221";
          }
        ];
      };
    };

    services.rpcbind.enable = true;

    fileSystems = {
      "/exports/work" = {
        device = "/work";
        options = [ "bind" ];
      };
    };

    programs.singularity.enable = true;

    boot.kernelParams = [
      "amd_iommu=on" "iommu=pt"
    ];
    boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_16;

    services.tailscale =  {
      enable = true;
      authKeyFile = "/var/lib/secrets/tailscale.key";
      useRoutingFeatures = "both"; # for exit-node usage
      extraUpFlags = [
        "--login-server=https://headscale.svc.oceanbox.io"
        "--accept-dns=true"
        "--accept-routes=true"
        "--advertise-routes=172.16.238.0/24,172.16.239.0/24"
        "--snat-subnet-routes=false"
      ];
    };

    services.networkd-dispatcher = {
      enable = true;
      rules = {
        "tailscale-router" = {
          onState = [ "routable" ];
          script = ''
            #!${pkgs.runtimeShell}
            ${pkgs.ethtool}/bin/ethtool -K enp65s0np0 rx-udp-gro-forwarding on rx-gro-list off
            exit 0
          '';
        };
      };
    };

    imports = [
        ./hardware-configuration.nix
        ../default.nix
        ../mounts.nix
    ];
  };
}