platform/clusters/stokes/cluster.nix

{ pkgs, lib, config, ... }:
with lib;
let
  cfg = config.features.host;

  mkSANs = host: [
    host.name
    host.address
    "127.0.0.1"
  ];

  configuration = {
    system.autoUpgrade.enable = lib.mkForce false;

    boot = {
      loader.systemd-boot.enable = true;
      loader.efi.canTouchEfiVariables = true;
      kernelPackages = pkgs.linuxPackages_5_4;
      kernelModules = [ "ib_umad" "ib_ipoib" ];
      kernelParams = [
        "console=ttyS0,115200"
        "console=tty0"
      ];
    };


    console = {
      font = "Lat2-Terminus16";
      keyMap = "us";
    };

    i18n = {
      defaultLocale = "en_DK.UTF-8";
      extraLocaleSettings = {
        LC_TIME = "en_DK.UTF-8";
      };
    };

    time.timeZone = "Europe/Oslo";

    features = {
      os = {
        # boot.uefi = true;
        adminAuthorizedKeys = [
          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiAS30ZO+wgfAqDE9Y7VhRunn2QszPHA5voUwo+fGOf jonas"
          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDULdlLC8ZLu9qBZUYsjhpr6kv5RH4yPkekXQdD7prkqapyoptUkO1nOTDwy7ZsKDxmp9Zc6OtdhgoJbowhGW3VIZPmooWO8twcaYDpkxEBLUehY/n8SlAwBtiHJ4mTLLcynJMVrjmTQLF3FeWVof0Aqy6UtZceFpLp1eNkiHTCM3anwtb9+gfr91dX1YsAOqxqv7ooRDu5rCRUvOi4OvRowepyuBcCjeWpTkJHkC9WGxuESvDV3CySWkGC2fF2LHkAu6SFsFE39UA5ZHo0b1TK+AFqRFiBAb7ULmtuno1yxhpBxbozf8+Yyc7yLfMNCyBpL1ci7WnjKkghQv7yM1xN2XMJLpF56v0slSKMoAs7ThoIlmkRm/6o3NCChgu0pkpNg/YP6A3HfYiEDgChvA6rAHX6+to50L9xF3ajqk4BUzWd/sCk7Q5Op2lzj31L53Ryg8vMP8hjDjYcgEcCCsGOcjUVgcsmfC9LupwRIEz3aF14AWg66+3zAxVho8ozjes= jonas.juselius@juselius.io"
          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCk5EKXxo/KLogjqSxSf/GkQdZ30UxB3wXc5k6Y6RRKQ/5iJ+XyYTbuqYOUp30p54apZzbayU2icahE/upr754lQicQwJtOXW/Iut57VRhSpq4P+mKCIdT58xCUkAZYr8Aja8UjHlYeJgFvp023K/fqmwbapu8R1gh4bzXm7uU1XeJoYfuOb+Cb8NGMn1ICrw2aztA0yVOXZ7tyJd2qyr1+6PuM/Ca2nKN4wLIX2vwyN3vZjR15nkIaHQGlTaJlNk2NEG1YTxsIQ9axDjNtyL80kjUr5M8zxW6s0h3451zr1b21EetP1i+1POIjS9uWXv5iabF+1Qb1GaS4FAYzzpqNY+moLzY7Zqfi05MPsMYkNoZ1Kg5aj0IuZb0OM9i6ZJrFs9nYAGG0uLSUTfrs957f9nokFyILGYg5xY46YN3uQrqfZifvcR0KaEdxEKvnfq0qrNG3uYLR/OYm2yblRcNbWgDoQ1hH7qa9uJM2JrPM07s4sJGkqfAib8Hwz9+l7jMrL6KIGUOA4aX0B1KZaIKKiZa42WlgdbeA17aW3laIqS5mZCkI3pLMYZAxe+A6rQi+V8ZAvDSyOL/Vws3lboXaN5QLu17R8uCY7MkIAvRBiZSpdWNeX3JO5m6zexkxkrFlxyEBf+ott4ATSw+eMYMs8i5xQRqPjgO1cABWkUdGpw== martin.moe.carstens@itpartner.no"
        ];
        docker.enable = false;

        mailRelay = {
          enable = true;
          adminEmail = "jonas.juselius@tromso.serit.no";
          mailDomain = "itpartner.no";
          mailGateway = "smtpgw.itpartner.no:465";
          mailAuthUser = "utvikling";
        };
      };
      cachix.enable = false;

      monitoring.nodeExporter.enable = false;

      pki = { ca = ./ca; };

      hpc = {
        enable = true;
        slurm = {
          client = true;
          mungeKey = ./munge.key;
          controlMachine = "stokes";
          nodeName = [
            "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
            "stokes Sockets=2 CoresPerSocket=16 ThreadsPerCore=2 RealMemory=64000 TmpDisk=500000 State=UNKNOWN"
          ];
          partitionName = [
            "batch Nodes=c0-[1-8] Default=YES MaxTime=INFINITE State=UP"
            "frontend Nodes=stokes MaxTime=1:00:00 State=UP"
          ];
        };
        beegfs = {
          enable = true;
          beegfs = {
            mds0-0 = {
              mgmtdHost = "mds0-0";
              connAuthFile = "";
              client = {
                enable = true;
                mountPoint = "/work";
              };
            };
          };
        };
      };

      k8s = {
        enable = true;
        node.enable = true;
        clusterName = "hpc0";
        initca = ./ca;
        cidr = "10.100.0.0/16";
        master = {
          name = "stokes";
          address = "10.1.61.100";
          extraSANs = [ "hpc0-0.regnekraft.io" ];
        };
        ingressNodes = [
          "hpc0-0.regnekraft.io"
        ];
        fileserver = "mds0-0";
        charts = {
          acme_email = "innovasjon@itpartner.no";
          grafana_smtp_user = "utvikling";
          grafana_smtp_password = "S0m3rp0m@de#21!";
        };
      };
    };

    services.kubernetes.kubelet.extraSANs = mkSANs {
      name = cfg.name;
      address = cfg.address;
    };

    networking = {
      domain = mkDefault "regnekraft.io";
      defaultGateway = mkDefault "10.1.61.1";
      nameservers = mkDefault [ "8.8.8.8" ];
      search = mkDefault [ "local" ];
      extraHosts = import ./hosts.nix;
      firewall.extraCommands = ''
        iptables -I INPUT -s 10.1.61.0/24 -j ACCEPT
      '';
    };

    environment.variables = {};

    systemd.services."serial-getty@ttyS0".enable = true;

    nix = {
      maxJobs = 32;
      trustedUsers = [ "@wheel" ];
      # binaryCachePublicKeys = [
      #   "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig=="
      # ];
    };
  };

  deployment = {
    deployment.targetHost = cfg.address;
  };

  i40efix = {
    boot = let kernelExtras = pkgs.callPackage ./kernel.nix {
      kernel = pkgs.linuxPackages_5_4.kernel;
    }; in {
      extraModulePackages = [ kernelExtras.i40e2 ];
      kernelModules = [ "ib_umad" "ib_ipoib" "i40e2" ];
    };
  };

  shosts = {
    environment.etc."ssh/shosts.equiv" = {
      mode = "0644";
      uid = 0;
      gid = 0;
      text = ''
        10.1.62.2
        10.1.61.100
        10.1.61.101
        10.1.61.102
        10.1.61.103
        10.1.61.104
        10.1.61.105
        10.1.61.106
        10.1.61.107
        10.1.61.108
        10.1.63.100
        10.1.63.101
        10.1.63.102
        10.1.63.103
        10.1.63.104
        10.1.63.105
        10.1.63.106
        10.1.63.107
        10.1.63.108
      '';
    };

    programs.ssh.knownHosts = {
      stokes = {
        hostNames = [
          "stokes" "stokes.hpc.local" "stokes.regnekraft.io" "10.1.61.100"
          "10.1.63.100" "10.1.62.2"
        ];
        publicKeyFile = ./pubkeys/stokes.pub;
      };
      c0-1 = { hostNames = [ "c0-1" "c0-1.hpc.local" "10.1.61.101" "10.1.63.101" ]; publicKeyFile = ./pubkeys/c0-1.pub; };
      c0-2 = { hostNames = [ "c0-2" "c0-2.hpc.local" "10.1.61.102" "10.1.63.102" ]; publicKeyFile = ./pubkeys/c0-2.pub; };
      c0-3 = { hostNames = [ "c0-3" "c0-3.hpc.local" "10.1.61.103" "10.1.63.103" ]; publicKeyFile = ./pubkeys/c0-3.pub; };
      c0-4 = { hostNames = [ "c0-4" "c0-4.hpc.local" "10.1.61.104" "10.1.63.104" ]; publicKeyFile = ./pubkeys/c0-4.pub; };
      c0-5 = { hostNames = [ "c0-5" "c0-5.hpc.local" "10.1.61.105" "10.1.63.105" ]; publicKeyFile = ./pubkeys/c0-5.pub; };
      c0-6 = { hostNames = [ "c0-6" "c0-6.hpc.local" "10.1.61.106" "10.1.63.106" ]; publicKeyFile = ./pubkeys/c0-6.pub; };
      c0-7 = { hostNames = [ "c0-7" "c0-7.hpc.local" "10.1.61.107" "10.1.63.107" ]; publicKeyFile = ./pubkeys/c0-7.pub; };
      c0-8 = { hostNames = [ "c0-8" "c0-8.hpc.local" "10.1.61.108" "10.1.63.108" ]; publicKeyFile = ./pubkeys/c0-8.pub; };
      };

      environment.systemPackages = [ openssh-shosts ];

      security.wrappers = {
        ssh-keysign = {
          source = "${openssh-shosts}/libexec/ssh-keysign";
          owner = "root";
          group = "root";
          permissions = "u+rs,g+rx,o+rx";
        };
      };
  };

  openssh-shosts = pkgs.openssh.overrideAttrs (attrs: {
    buildFlags = [ "SSH_KEYSIGN=/run/wrappers/bin/ssh-keysign" ];
  });

  myvnc =
    let
      myvnc = pkgs.writeScriptBin "myvnc" ''
        #!${pkgs.runtimeShell}

        uid=`id -u`
        port=$((9000+$uid))
        shell=`getent passwd $(id -un) | awk -F : '{print $NF}'`
        # vnc=${pkgs.tigervnc}/bin/vncserver
        vnc=/nix/store/czp2b60dwk75widi8y287hr0xx1wgv2a-tigervnc-1.10.1/bin/vncserver

        case $1 in
        -p|--port) shift; port=$1 ;;
        kill|stop)
            display=$($vnc -list | sed  -n 's/^\(:[0-9]\+\).*/\1/p'| head -1)
            $vnc -kill $display
            exit 0
            ;;
        esac
        ps ax | sed '/grep/d' | grep "Xvnc.*-rfbport $port" >/dev/null 2>&1
        [ $? = 1 ] && $vnc -rfbport $port
        echo "Xvnc server is running on port $port."
        exec $shell -i
      '';

      buildCommand = ''
          mkdir -p $out/bin
          echo $src > $out/bin/myvnc
          chmod 755 $out/bin/myvnc
      '';
    in {
      environment.systemPackages = [ myvnc ];
    };

in {
  options.node = {
    i40efix = mkEnableOption "Apply fix for i40e driver";

    myvnc = mkEnableOption "Enable myvnc script";
  };

  config = mkMerge [
    configuration

    deployment

    shosts

    (mkIf config.node.i40efix i40efix)

    (mkIf config.node.myvnc myvnc)
  ];

  imports = [
    ../../modules
    ../../nixos
    ./users.nix
  ];
}