platform/clusters/stokes/morph.nix

let
  # Pin the deployment package-set to a specific version of nixpkgs
  # pkgs = import (builtins.fetchTarball {
  #   url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
  #   sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
  # }) {};
  pkgs = import <nixpkgs> {};

  etcdNodes = {
    # hpc0-0 = "10.1.63.100";
    # hpc0-1 = "10.1.63.101";
    # hpc0-2 = "10.1.63.102";
  };

  etcdCluster = {
    enable = false;
    existing = false;
    nodes = etcdNodes;
  };

  k8sNodes = [
    # { name = "hpc0-1"; address = "10.1.61.101"; }
  ];

  stokes = {
    deployment.tags = [ "frontend" ];
    node.myvnc = true;

    systemd.targets = {
      sleep.enable = false;
      suspend.enable = false;
      hibernate.enable = false;
      hybrid-sleep.enable = false;
    };

    features = {
      host = {
        address = "10.1.61.100";
        name = "hpc0-0";
      };
      os = {
        externalInterface = "eno1";
        nfs.enable = true;
        nfs.exports = ''
          /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
          /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
        '';
      };

      hpc = {
        slurm.server = true;
        frontend = true;
      };

      k8s = {
        enable = true;
        master.enable = true;
        node.enable = true;
        nodes = nodes;
        clusterName = "hpc0";
        initca = ./ca;
        cidr = "10.100.0.0/16";
        master = {
          name = "hpc0-0";
          address = "10.1.63.100";
          extraSANs = [ "stokes.regnekraft.io" ];
        };
        ingressNodes = [
          "hpc0-0.itpartner.intern"
        ];
        fileserver = "mds0-0";
        charts = {
          acme_email = "innovasjon@itpartner.no";
          grafana_smtp_user = "utvikling";
          grafana_smtp_password = "S0m3rp0m@de#21!";
        };
      };

      monitoring = {
        server = {
          enable = true;
          scrapeHosts = [
            "frontend" "mds0-0"
            "c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8"
          ];
          defaultAlertReceiver = {
            email_configs = [
              { to = "jonas.juselius@tromso.serit.no"; }
            ];
          };
          pageAlertReceiver = {
            webhook_configs = [
              {
                url = "https://prometheus-msteams.k2.itpartner.no/stokes";
                http_config = {
                  tls_config = { insecure_skip_verify = true; };
                };
              }
            ];
          };
        };
        webUI.enable = true;
        webUI.acmeEmail = "innovasjon@itpartner.no";
        webUI.allow = [
            "10.1.2.0/24"
            "172.19.254.0/24"
            "172.19.255.0/24"
        ];
        infiniband-exporter = {
          enable = true;
          nameMap = ''
            0x0c42a10300ddc4bc "frontend"
            0x1c34da0300787798 "mds0-0"
            0x0c42a10300dbe7f4 "c0-1"
            0x0c42a10300dbe7d8 "c0-2"
            0x0c42a10300dbe800 "c0-3"
            0x0c42a10300dbec80 "c0-4"
            0x0c42a10300dbea50 "c0-5"
            0x0c42a10300dbeb2c "c0-6"
            0x0c42a10300dbe7fc "c0-7"
            0x0c42a10300dbe5a0 "c0-8"
          '';
        };
        slurm-exporter = {
          enable = true;
          port = 6080;
        };
      };
    };

    networking = {
      useDHCP = false;
      interfaces.eno1 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.1.62.2";
          prefixLength = 24;
        } ];
      };
      interfaces.enp175s0f0 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.1.61.100";
          prefixLength = 24;
        } ];
      };
      interfaces.ibp59s0 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.1.63.100";
          prefixLength = 24;
        } ];
      };
      defaultGateway = "10.1.62.1";
      firewall.extraCommands = ''
        iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT
        iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE
      '';
    };

    fileSystems ={
      "/exports/home" = {
        device = "/home";
        options = [ "bind" ];
      };
      "/exports/opt" = {
        device = "/opt";
        options = [ "bind" ];
      };
      "/data" = {
        device = "10.1.63.80:/data";
        fsType = "nfs";
      };
    };

    security.pam.services.sshd.googleAuthenticator.enable = true;

    nix.extraOptions = ''
      secret-key-files = /etc/nix/stokes.private
    '';

    services.xserver = {
      enable = true;
      enableCtrlAltBackspace = true;
      layout = "us";
      xkbVariant = "altgr-intl";
      xkbOptions = "eurosign:e";
      displayManager = {
        gdm.enable = true;
        job.logToFile = true;
      };
      desktopManager.xfce.enable = true;
    };

    services.prometheus.alertmanager.configuration.global = {
        smtp_smarthost = "smtpgw.itpartner.no:465";
        smtp_auth_username = "utvikling";
        smtp_auth_password = "S0m3rp0m@de#21!";
        smtp_hello = "stokes.regnekraft.io";
        smtp_from = "noreply@stokes.regnekraft.io";
    };

    services.nginx = {
      virtualHosts = {
        "ds.matnoc.regnekraft.io" = {
          forceSSL = true;
          enableACME = true;
          serverAliases = [];
          locations."/" = {
            proxyPass = "http://localhost:9088";
            proxyWebsockets = false;
            extraConfig = ''
              allow 10.1.2.0/24;
              allow 172.19.254.0/24;
              allow 172.19.255.0/24;
              deny all;
            '';
          };
        };
      };
    };

    services.minio = {
      enable = true;
      region = "hpc";
      browser = true;
      accessKey = "admin";
      secretKey = "en to tre fire";
      listenAddress = "0.0.0.0:9000";
      dataDir = [ "/data/minio" ];
    };

    imports = [ ./cluster.nix ./hw/frontend.nix ];
  };

  compute = {
    deployment.tags = [ "compute" ];

    features = {
      os.externalInterface = "eno33";
      hpc.compute = true;
    };

    fileSystems = {
      "/home/stokes" = {
        device = "10.1.63.100:/home";
        fsType = "nfs";
      };
      "/opt" = {
        device = "10.1.63.100:/opt";
        fsType = "nfs";
      };
      "/data" = {
        device = "10.1.63.80:/data";
        fsType = "nfs";
      };
    };
  };

  genComputeNodes = idx: nNodes:
    let
      nodeList = builtins.genList (x: x + 1) nNodes;
      mkCompute = n:
        let
           ip    = "10.1.61.${toString (n + 100)}";
           ipoib = "10.1.63.${toString (n + 100)}";
           name = "c${toString idx}-${toString n}";
           k8sName = "hpc${toString idx}-${toString n}";
           hw = ./hw + "/${name}.nix";
        in {
          "${name}" = {
            node = {
              i40efix = true;
            };
            features.host = {
              address = ip;
              name = k8sName;
            };
            networking = {
              useDHCP = false;
              interfaces.eno33 = {
                useDHCP = false;
                ipv4.addresses = [ {
                  address = ip;
                  prefixLength = 24;
                } ];
                ipv4.routes = [ {
                  address = "10.1.62.2";
                  prefixLength = 32;
                  via = "10.1.61.100";
                } ];

              };
              interfaces.ibp65s0 = {
                useDHCP = false;
                ipv4.addresses = [ {
                  address = ipoib;
                  prefixLength = 24;
                } ];
              };
            };
            imports = [ ./cluster.nix hw ];
          } // compute;
        };
    in
      builtins.foldl' (a: n: a // mkCompute n) {} nodeList;
in
{
  network =  {
    inherit pkgs;
    description = "stokes";
    ordering = {
      tags = [ "frontend" "compute" ];
    };
  };

  inherit stokes;
} // genComputeNodes 0 8