platform/cluster/fs-work/default.nix

{ pkgs ? import <nixpkgs> {} }:
let
  # Pin the deployment package-set to a specific version of nixpkgs
  # pkgs = import (builtins.fetchTarball {
  #   url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
  #   sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
  # }) {};
  # pkgs = import <nixpkgs> {};
  etcdCluster = import ../etcdCluster.nix;
  name = "fs-work";
  address = "10.255.241.90";
in {
  fs-work = { config, pkgs, ... }: with pkgs; {
    deployment.tags = [ "fs" "fs-work" ];
    deployment.targetHost = address;
    system.autoUpgrade.enable = lib.mkForce false;

    systemd.targets = {
      sleep.enable = false;
      suspend.enable = false;
      hibernate.enable = false;
      hybrid-sleep.enable = false;
    };

    # services.udev.extraRules = ''
    #     KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
    # '';

    environment.systemPackages = with pkgs; [
        rdma-core
        hwloc
    ];

    cluster = {
      k8sNode = true;
      slurm = false;
      mounts = {
        rdma.enable = true;
        automount.enable = true;
        home = true;
        opt = false;
        work = false;
        data = false;
        backup = false;
        ceph = false;
      };
    };

    features.hpc.slurm.mungeUid = 994;

    features = {
      host = {
        inherit address;
        inherit name;
      };

      os = {
        externalInterface = "enp33s0f3np3";
        nfs.enable = true;
        nfs.exports = ''
          /exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
          /exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
          /exports 10.255.244.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
        '';
      };

      k8s = {
          enable = true;
          node.enable = true;
          master.enable = false;
          inherit etcdCluster;
      };
    };

    systemd.services.rc-local = {
        description = "rc.local script";
        wantedBy = [ "multi-user.target" ];
        after = [ "network.target" ];
        path = [ "/run/current-system/sw/" ];
        serviceConfig = {
          Type = "oneshot";
        };
        script = ''
            # if [ -e /sys/block/md126 ]; then
            #     echo "deadline" > /sys/block/md126/queue/scheduler
            #     # echo "4096" > /sys/block/md126/queue/nr_requests
            #     echo "4096" > /sys/block/md126/queue/read_ahead_kb
            #     echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
            #     echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
            # fi
            grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
            grep -q tcp /proc/fs/nfsd/portlist || echo "tcp 2049" > /proc/fs/nfsd/portlist
        '';
    };

    boot.kernel.sysctl = {
        "vm.dirty_background_ratio" = 5;
        "vm.dirty_ratio" = 10;
        "vm.vfs_cache_pressure" = 50;
        "vm.min_free_kbytes" = 262144;
    };

    networking = {
      hostName = name;
      interfaces.enp65s0f0np0 = {
        useDHCP = false;
        ipv4.addresses = [
          {
            address = address;
            prefixLength = 24;
          }
        ];
        ipv4.routes = [
            {
                address = "10.255.242.0";
                prefixLength = 24;
                via = "10.255.241.100";
            }
        ];
      };
      interfaces.enp1s0f1np1 = {
        useDHCP = false;
        ipv4.addresses = [
          {
            address = "10.255.244.90";
            prefixLength = 24;
          }
        ];
      };
      # interfaces."ibp65s0.7666" = {
      #   useDHCP = false;
      # };
      interfaces.ibp1s0f0 = {
        useDHCP = false;
        ipv4.addresses = [
          {
            address = "10.255.243.90";
            prefixLength = 24;
          }
      ];
      };
      firewall = {
        allowedTCPPorts = [];
        allowedUDPPorts = [];
        extraCommands = ''
          # iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
        '';
      };
    };

    services.rpcbind.enable = true;

    fileSystems = {
      "/exports/work" = {
        device = "/work";
        options = [ "bind" ];
      };
      "/exports/opt" = {
        device = "/opt";
        options = [ "bind" ];
      };
    };

    programs.singularity.enable = true;

    imports = [
        ./hardware-configuration.nix
        ../cluster.nix
        ../mounts.nix
    ];
  };
}