From 2d5b2d1e9d07b411a0c02ee7b1ff71c4c1b18e26 Mon Sep 17 00:00:00 2001 From: Jonas Juselius Date: Thu, 1 Jul 2021 12:40:56 +0200 Subject: [PATCH] Add deployment expr for morph for stokes --- clusters/stokes/stokes.nix | 263 +++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 clusters/stokes/stokes.nix diff --git a/clusters/stokes/stokes.nix b/clusters/stokes/stokes.nix new file mode 100644 index 0000000..39b6671 --- /dev/null +++ b/clusters/stokes/stokes.nix @@ -0,0 +1,263 @@ +let + # Pin the deployment package-set to a specific version of nixpkgs + # pkgs = import (builtins.fetchTarball { + # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; + # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; + # }) {}; + pkgs = import {}; + + stokes = { + deployment.tags = [ "frontend" ]; + node.address = "10.1.62.2"; + node.myvnc = true; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + features = { + os = { + externalInterface = "eno1"; + nfs.enable = true; + nfs.exports = '' + /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + hpc = { + slurm.server = true; + frontend = true; + }; + + monitoring = { + server = { + enable = true; + scrapeHosts = [ + "frontend" "mds0-0" + "c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8" + ]; + defaultAlertReceiver = { + email_configs = [ + { to = "jonas.juselius@tromso.serit.no"; } + ]; + }; + pageAlertReceiver = { + webhook_configs = [ + { + url = "https://prometheus-msteams.k2.itpartner.no/stokes"; + http_config = { + tls_config = { insecure_skip_verify = true; }; + }; + } + ]; + }; + }; + webUI.enable = true; + webUI.acmeEmail = "innovasjon@itpartner.no"; + webUI.allow = [ + "10.1.2.0/24" + "172.19.254.0/24" + "172.19.255.0/24" + ]; + infiniband-exporter = { + enable = true; + nameMap = '' + 0x0c42a10300ddc4bc "frontend" + 0x1c34da0300787798 "mds0-0" + 0x0c42a10300dbe7f4 "c0-1" + 0x0c42a10300dbe7d8 "c0-2" + 0x0c42a10300dbe800 "c0-3" + 0x0c42a10300dbec80 "c0-4" + 0x0c42a10300dbea50 "c0-5" + 0x0c42a10300dbeb2c "c0-6" + 0x0c42a10300dbe7fc "c0-7" + 0x0c42a10300dbe5a0 "c0-8" + ''; + }; + slurm-exporter = { + enable = true; + port = 6080; + }; + }; + }; + + networking = { + useDHCP = false; + interfaces.eno1 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.62.2"; + prefixLength = 24; + } ]; + }; + interfaces.enp175s0f0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.61.100"; + prefixLength = 24; + } ]; + }; + interfaces.ibp59s0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.63.100"; + prefixLength = 24; + } ]; + }; + defaultGateway = "10.1.62.1"; + firewall.extraCommands = '' + iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE + ''; + }; + + fileSystems ={ + "/exports/home" = { + device = "/home"; + options = [ "bind" ]; + }; + "/exports/opt" = { + device = "/opt"; + options = [ "bind" ]; + }; + "/data" = { + device = "10.1.63.80:/data"; + fsType = "nfs"; + }; + }; + + security.pam.services.sshd.googleAuthenticator.enable = true; + + nix.extraOptions = '' + secret-key-files = /etc/nix/stokes.private + ''; + + services.xserver = { + enable = true; + enableCtrlAltBackspace = true; + layout = "us"; + xkbVariant = "altgr-intl"; + xkbOptions = "eurosign:e"; + displayManager = { + gdm.enable = true; + job.logToFile = true; + }; + desktopManager.xfce.enable = true; + }; + + services.prometheus.alertmanager.configuration.global = { + smtp_smarthost = "smtpgw.itpartner.no:465"; + smtp_auth_username = "utvikling"; + smtp_auth_password = "S0m3rp0m@de#21!"; + smtp_hello = "stokes.regnekraft.io"; + smtp_from = "noreply@stokes.regnekraft.io"; + }; + + services.nginx = { + virtualHosts = { + "ds.matnoc.regnekraft.io" = { + forceSSL = true; + enableACME = true; + serverAliases = []; + locations."/" = { + proxyPass = "http://localhost:9088"; + proxyWebsockets = false; + extraConfig = '' + allow 10.1.2.0/24; + allow 172.19.254.0/24; + allow 172.19.255.0/24; + deny all; + ''; + }; + }; + }; + }; + + + imports = [ ./cluster.nix ./hw/frontend.nix ]; + }; + + compute = { + deployment.tags = [ "compute" ]; + + features = { + os.externalInterface = "eno33"; + hpc.compute = true; + }; + + fileSystems = { + "/home/stokes" = { + device = "10.1.63.100:/home"; + fsType = "nfs"; + }; + "/opt" = { + device = "10.1.63.100:/opt"; + fsType = "nfs"; + }; + "/data" = { + device = "10.1.63.80:/data"; + fsType = "nfs"; + }; + }; + }; + + genComputeNodes = idx: nNodes: + let + nodeList = builtins.genList (x: x + 1) nNodes; + mkCompute = n: + let + ip = "10.1.61.${toString (n + 100)}"; + ipoib = "10.1.63.${toString (n + 100)}"; + name = "c${toString idx}-${toString n}"; + hw = ./hw + "/${name}.nix"; + in { + "${name}" = { + node = { + address = ip; + i40efix = true; + }; + networking = { + useDHCP = false; + interfaces.eno33 = { + useDHCP = false; + ipv4.addresses = [ { + address = ip; + prefixLength = 24; + } ]; + ipv4.routes = [ { + address = "10.1.62.2"; + prefixLength = 32; + via = "10.1.61.100"; + } ]; + + }; + interfaces.ibp65s0 = { + useDHCP = false; + ipv4.addresses = [ { + address = ipoib; + prefixLength = 24; + } ]; + }; + }; + imports = [ ./cluster.nix hw ]; + } // compute; + }; + in + builtins.foldl' (a: n: a // mkCompute n) {} nodeList; +in +{ + network = { + inherit pkgs; + description = "stokes"; + ordering = { + tags = [ "frontend" "compute" ]; + }; + }; + + inherit stokes; +} // genComputeNodes 0 8 +