platform/configuration.nix

{ pkgs, ...}:
let
  computeNodes = import ./cluster/compute/nodes.nix;
  etcdCluster = import ./cluster/etcdCluster.nix;
  myvnc = import ./cluster/compute/myvnc.nix { inherit pkgs; };
in
{
    systemd.targets = {
      sleep.enable = false;
      suspend.enable = false;
      hibernate.enable = false;
      hybrid-sleep.enable = false;
    };

    cluster = {
      compute = true;
      k8sNode = true;
    };

    features = {
      host = {
        address = "10.255.241.100";
        name = "ekman";
      };

      os = {
        externalInterface = "enp33s0f0np0";
        nfs.enable = true;
        nfs.exports = ''
          /exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
          /exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
        '';
      };

      hpc = {
        slurm.server = true;
        frontend = true;
      };

      k8s = {
        master.enable = true;
        node.enable = true;
        nodes = computeNodes;
        inherit etcdCluster;
      };

      monitoring = {
        server = {
          enable = false;
          scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
          defaultAlertReceiver = {
            email_configs = [
              { to = "jonas.juselius@oceanbox.io"; }
            ];
          };
          pageAlertReceiver = {
            webhook_configs = [
              {
                url = "https://prometheus-msteams.k2.itpartner.no/ekman";
                http_config = {
                  tls_config = { insecure_skip_verify = true; };
                };
              }
            ];
          };
        };
        webUI.enable = false;
        webUI.acmeEmail = "innovasjon@itpartner.no";
        webUI.allow = [
            "10.1.2.0/24"
            "172.19.254.0/24"
            "172.19.255.0/24"
        ];
        infiniband-exporter = {
          enable = true;
          nameMap = ''
            0xe8ebd3030024a2c6 "frontend"
            0xe8ebd3030024981e "c0-1"
            0xe8ebd3030024a21a "c0-2"
       	    0xe8ebd30300249a3a "c0-3"
            0xe8ebd30300248b9e "c0-4"
            0xe8ebd30300248b86 "c0-5"
            0xe8ebd3030024998a "c0-6"
            0xe8ebd30300248b8e "c0-7"
            0xe8ebd3030024999e "c0-8"
            0xe8ebd30300248fca "c0-9"
            0xe8ebd3030024a216 "c0-10"
            0xe8ebd30300248b96 "c0-11"
            0xe8ebd30300248b9a "c0-12"
            0xe8ebd303002495d2 "c0-13"
            0xe8ebd303002495e2 "c0-14"
            0xe8ebd30300248f42 "c0-15"
            0xe8ebd303002495e6 "c0-16"
            0xe8ebd3030024a2a2 "nfs0"
            0xe8ebd3030024a2ae "nfs1"
          '';
        };
        slurm-exporter = {
          enable = true;
          port = 6080;
        };
      };
    };

    services.udev.extraRules = ''
        KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
    '';

    networking = {
      useDHCP = false;
      hostName = "ekman";
      interfaces.enp33s0f0np0 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.255.242.2";
          prefixLength = 24;
        } ];
      };
      interfaces.enp33s0f3np3 = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.255.241.100";
          prefixLength = 24;
        } ];
        # ipv4.routes = [ {
        #   address = "10.1.2.0";
        #   prefixLength = 24;
        #   via = "10.255.241.1";
        # } ];
      };
      interfaces."ibp65s0" = {
        useDHCP = false;
      };
      interfaces."ibp65s0.7666" = {
        useDHCP = false;
        ipv4.addresses = [ {
          address = "10.255.243.100";
          prefixLength = 24;
        } ];
      };
      defaultGateway = "10.255.242.1";
      firewall = {
        allowedTCPPorts = [ 4443 ];
        extraCommands = ''
          # iptables -I INPUT -s 10.1.2.0/22 -j ACCEPT
          iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
          iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
          iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
        '';
      };
    };

    fileSystems = {
      "/exports/home" = {
        device = "/home";
        options = [ "bind" ];
      };
      "/frontend" = {
        device = "/home";
        options = [ "bind" ];
      };
      "/vol/local-storage/vol1" = {
        device = "/vol/vol1";
        options = [ "bind" ];
      };
      "/vol/local-storage/vol2" = {
        device = "/vol/vol2";
        options = [ "bind" ];
      };
    };

    nix.extraOptions = ''
      secret-key-files = /etc/nix/ekman.key
    '';

    services.xserver = {
      enable = true;
      enableCtrlAltBackspace = true;
      layout = "us";
      xkbVariant = "altgr-intl";
      xkbOptions = "eurosign:e";
      displayManager = {
        gdm.enable = true;
        job.logToFile = true;
      };
      desktopManager.xfce.enable = true;
    };

    services.prometheus.alertmanager.configuration.global = {
        smtp_smarthost = "smtpgw.itpartner.no";
        # smtp_auth_username = "utvikling";
        # smtp_auth_password = "S0m3rp0m@de#21!";
        smtp_hello = "ekman.oceanbox.io";
        smtp_from = "noreply@ekman.oceanbox.io";
    };

    # services.nginx = {
    #   virtualHosts = {
    #     "ds.matnoc.regnekraft.io" = {
    #       forceSSL = true;
    #       enableACME = true;
    #       serverAliases = [];
    #       locations."/" = {
    #         proxyPass = "http://localhost:9088";
    #         proxyWebsockets = false;
    #         extraConfig = ''
    #           allow 10.1.2.0/24;
    #           allow 172.19.254.0/24;
    #           allow 172.19.255.0/24;
    #           deny all;
    #         '';
    #       };
    #     };
    #   };
    # };

    # services.gitlab-runner = {
    #   enable = true;
    #   extraPackages = with pkgs; [
    #     singularity
    #   ];
    #   concurrent = 4;
    #   services = {
    #     sif = {
    #       registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
    #       executor = "shell";
    #       tagList = [ "ekman" "sif" ];
    #     };
    #   };
    # };

    # security.sudo.extraConfig = ''
    #   gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
    # '';

    security.pam = {
      services.sshd.googleAuthenticator.enable = true;
      loginLimits = [
        {
          domain = "@users";
          item = "rss";
          type = "hard";
          value = 16000000;
        }
        {
          domain = "@users";
          item = "cpu";
          type = "hard";
          value = 180;
        }
      ];
    };

    system.activationScripts = {
        home-permissions.text = ''
            chmod 755 /home/olean
            chmod 755 /home/frankgaa
            chmod 755 /home/jonas
            chmod 755 /home/stig
            chmod 755 /home/bast
            chmod 755 /home/simenlk
        '';
    };


    # ssh-rsa is deprecated, but putty/winscp users use it
    # services.openssh.extraConfig = ''
    #   pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
    # '';

    environment.systemPackages = [ myvnc ];

    imports = [ ./cluster/cluster.nix ./hardware-configuration.nix ];
  }