platform/clusters/stokes/default.nix

let
  # Pin the deployment package-set to a specific version of nixpkgs
  # pkgs = import (builtins.fetchTarball {
  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
  # }) {};
  pkgs = import <nixpkgs> {};

  etcdNodes = {
    c0-0 = "10.1.61.100";
    c0-1 = "10.1.61.101";
    c0-2 = "10.1.61.102";
  };

  etcdCluster = {
    enable = true;
    existing = true;
    nodes = etcdNodes;
  };

  nodes =
    with builtins;
    let nodes = genList (n: n + 1) 8; in
    map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;

  stokes = {
    # deployment.tags = [ "frontend" ];
    node.myvnc = true;

    systemd.targets = {
      sleep.enable = false;
      suspend.enable = false;
      hibernate.enable = false;
      hybrid-sleep.enable = false;
    };

    features = {
      host = {
        address = "10.1.62.2";
        name = "c0-0";
      };

      os = {
        externalInterface = "eno1";
        nfs.enable = true;
        nfs.exports = ''
          /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
          /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
        '';
      };

      hpc = {
        slurm.server = true;
        frontend = true;
      };

      k8s = {
        master.enable = true;
        node.enable = true;
        inherit nodes;
        inherit etcdCluster;
      };

      monitoring = {
        server = {
          enable = false;
          scrapeHosts = [ "frontend" "mds0-0" ] ++ (builtins.map (x: x.name) nodes);
          defaultAlertReceiver = {
            email_configs = [
              { to = "jonas.juselius@tromso.serit.no"; }
            ];
          };
          pageAlertReceiver = {
            webhook_configs = [
              {
                url = "https://prometheus-msteams.k2.itpartner.no/stokes";
                http_config = {
                  tls_config = { insecure_skip_verify = true; };
                };
              }
            ];
          };
        };
        webUI.enable = false;
        webUI.acmeEmail = "innovasjon@itpartner.no";
        webUI.allow = [
            "10.1.2.0/24"
            "172.19.254.0/24"
            "172.19.255.0/24"
        ];
        infiniband-exporter = {
          enable = true;
          nameMap = ''
            0x0c42a10300ddc4bc "frontend"
            0x1c34da0300787798 "mds0-0"
            0x0c42a10300dbe7f4 "c0-1"
            0x0c42a10300dbe7d8 "c0-2"
            0x0c42a10300dbe800 "c0-3"
            0x0c42a10300dbec80 "c0-4"
            0x0c42a10300dbea50 "c0-5"
            0x0c42a10300dbeb2c "c0-6"
            0x0c42a10300dbe7fc "c0-7"
            0x0c42a10300dbe5a0 "c0-8"
          '';
        };
        slurm-exporter = {
          enable = true;
          port = 6080;
        };
      };
    };

    networking = {
      useDHCP = false;
      interfaces.eno1 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.1.62.2";
          prefixLength = 24;
        } ];
      };
      interfaces.enp175s0f0 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.1.61.100";
          prefixLength = 24;
        } ];
      };
      interfaces.ibp59s0 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.1.63.100";
          prefixLength = 24;
        } ];
      };
      defaultGateway = "10.1.62.1";
      firewall.extraCommands = ''
        iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT
        iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE
      '';
    };

    fileSystems ={
      "/exports/home" = {
        device = "/home";
        options = [ "bind" ];
      };
      "/exports/opt" = {
        device = "/opt";
        options = [ "bind" ];
      };
      "/data" = {
        device = "10.1.63.80:/data";
        fsType = "nfs";
      };
      "/vol/local-storage/vol1" = {
        device = "/vol/vol1";
        options = [ "bind" ];
      };
      "/vol/local-storage/vol2" = {
        device = "/vol/vol2";
        options = [ "bind" ];
      };
    };

    nix.extraOptions = ''
      secret-key-files = /etc/nix/stokes.private
    '';

    services.xserver = {
      enable = true;
      enableCtrlAltBackspace = true;
      layout = "us";
      xkbVariant = "altgr-intl";
      xkbOptions = "eurosign:e";
      displayManager = {
        gdm.enable = true;
        job.logToFile = true;
      };
      desktopManager.xfce.enable = true;
    };

    services.prometheus.alertmanager.configuration.global = {
        smtp_smarthost = "smtpgw.itpartner.no:465";
        smtp_auth_username = "utvikling";
        smtp_auth_password = "S0m3rp0m@de#21!";
        smtp_hello = "stokes.regnekraft.io";
        smtp_from = "noreply@stokes.regnekraft.io";
    };

    services.nginx = {
      virtualHosts = {
        "ds.matnoc.regnekraft.io" = {
          forceSSL = true;
          enableACME = true;
          serverAliases = [];
          locations."/" = {
            proxyPass = "http://localhost:9088";
            proxyWebsockets = false;
            extraConfig = ''
              allow 10.1.2.0/24;
              allow 172.19.254.0/24;
              allow 172.19.255.0/24;
              deny all;
            '';
          };
        };
      };
    };

    # services.gitlab-runner = {
    #   enable = true;
    #   extraPackages = with pkgs; [
    #     singularity
    #   ];
    #   concurrent = 4;
    #   services = {
    #     sif = {
    #       registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
    #       executor = "shell";
    #       tagList = [ "stokes" "sif" ];
    #     };
    #   };
    # };

    # security.sudo.extraConfig = ''
    #   gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
    # '';

    security.pam = {
      services.sshd.googleAuthenticator.enable = true;
      loginLimits = [
        {
          domain = "@users";
          item = "rss";
          type = "hard";
          value = 16000000;
        }
        {
          domain = "@users";
          item = "cpu";
          type = "hard";
          value = 180;
        }
      ];
    };

    imports = [ ./cluster.nix ./hw/frontend.nix ];
  };

  compute = {
    # deployment.tags = [ "compute" ];

    fileSystems = {
      "/home/stokes" = {
        device = "10.1.63.100:/home";
        fsType = "nfs";
      };
      "/opt" = {
        device = "10.1.63.100:/opt";
        fsType = "nfs";
      };
      "/data" = {
        device = "10.1.63.80:/data";
        fsType = "nfs";
      };
    };
  };

  mkCompute = host:
  let
    ipoib = builtins.replaceStrings [".61."] [".63."] host.address;
    hw = ./hw + "/${host.name}.nix";
  in {
    "${host.name}" = {
      features = {
        inherit host;
        os.externalInterface = "eno33";
        hpc.compute = true;
        k8s = { inherit etcdCluster; };
      };

      node = {
        i40efix = true;
      };

      networking = {
        useDHCP = false;
        interfaces.eno33 = {
          useDHCP = false;
          ipv4.addresses = [ {
            address = host.address;
            prefixLength = 24;
          } ];
          ipv4.routes = [ {
            address = "10.1.62.2";
            prefixLength = 32;
            via = "10.1.61.100";
          } ];

        };
        interfaces.ibp65s0 = {
          useDHCP = false;
          ipv4.addresses = [ {
            address = ipoib;
            prefixLength = 24;
          } ];
        };
      };
      imports = [ ./cluster.nix hw ];
    }
    // compute;
};
in {
    ## morph
    # network =  {
    #   inherit pkgs;
    #   description = "stokes";
    #   ordering = {
    #     tags = [ "frontend" "compute" ];
    #   };
    # };

    inherit stokes;
} // builtins.foldl' (a: n: a // mkCompute n) {} nodes