diff --git a/clusters/stokes/cluster.nix b/clusters/stokes/cluster.nix index 1ee8e3e..125f056 100644 --- a/clusters/stokes/cluster.nix +++ b/clusters/stokes/cluster.nix @@ -3,6 +3,17 @@ with lib; let cfg = config.features.host; + etcdCluster = { + enable = true; + existing = true; + nodes = + { + c0-0 = "10.1.61.100"; + c0-1 = "10.1.61.101"; + c0-2 = "10.1.61.102"; + }; + }; + mkSANs = host: [ host.name host.address @@ -101,6 +112,7 @@ let slurm = { client = true; mungeKey = ./munge.key; + mungeUid = 996; # hack controlMachine = "stokes"; nodeName = [ "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN" @@ -130,6 +142,7 @@ let enable = true; node.enable = true; clusterName = "hpc0"; + inherit etcdCluster; initca = ./ca; cidr = "10.100.0.0/16"; master = { diff --git a/clusters/stokes/default.nix b/clusters/stokes/default.nix index 306bc04..4309bd7 100644 --- a/clusters/stokes/default.nix +++ b/clusters/stokes/default.nix @@ -6,257 +6,14 @@ let # }) {}; pkgs = import {}; - etcdNodes = { - c0-0 = "10.1.61.100"; - c0-1 = "10.1.61.101"; - c0-2 = "10.1.61.102"; - }; - - etcdCluster = { - enable = true; - existing = true; - nodes = etcdNodes; - }; - nodes = with builtins; let nodes = genList (n: n + 1) 8; in map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes; - stokes = { - # deployment.tags = [ "frontend" ]; - node.myvnc = true; - - systemd.targets = { - sleep.enable = false; - suspend.enable = false; - hibernate.enable = false; - hybrid-sleep.enable = false; - }; - - features = { - host = { - address = "10.1.62.2"; - name = "c0-0"; - }; - - os = { - externalInterface = "eno1"; - nfs.enable = true; - nfs.exports = '' - /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - ''; - }; - - hpc = { - slurm.server = true; - frontend = true; - }; - - k8s = { - master.enable = true; - node.enable = true; - inherit nodes; - inherit etcdCluster; - }; - - monitoring = { - server = { - enable = false; - scrapeHosts = [ "frontend" "mds0-0" ] ++ (builtins.map (x: x.name) nodes); - defaultAlertReceiver = { - email_configs = [ - { to = "jonas.juselius@tromso.serit.no"; } - ]; - }; - pageAlertReceiver = { - webhook_configs = [ - { - url = "https://prometheus-msteams.k2.itpartner.no/stokes"; - http_config = { - tls_config = { insecure_skip_verify = true; }; - }; - } - ]; - }; - }; - webUI.enable = false; - webUI.acmeEmail = "innovasjon@itpartner.no"; - webUI.allow = [ - "10.1.2.0/24" - "172.19.254.0/24" - "172.19.255.0/24" - ]; - infiniband-exporter = { - enable = true; - nameMap = '' - 0x0c42a10300ddc4bc "frontend" - 0x1c34da0300787798 "mds0-0" - 0x0c42a10300dbe7f4 "c0-1" - 0x0c42a10300dbe7d8 "c0-2" - 0x0c42a10300dbe800 "c0-3" - 0x0c42a10300dbec80 "c0-4" - 0x0c42a10300dbea50 "c0-5" - 0x0c42a10300dbeb2c "c0-6" - 0x0c42a10300dbe7fc "c0-7" - 0x0c42a10300dbe5a0 "c0-8" - ''; - }; - slurm-exporter = { - enable = true; - port = 6080; - }; - }; - }; - - networking = { - useDHCP = false; - interfaces.eno1 = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.1.62.2"; - prefixLength = 24; - } ]; - }; - interfaces.enp175s0f0 = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.1.61.100"; - prefixLength = 24; - } ]; - }; - interfaces.ibp59s0 = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.1.63.100"; - prefixLength = 24; - } ]; - }; - defaultGateway = "10.1.62.1"; - firewall.extraCommands = '' - iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT - iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE - ''; - }; - - fileSystems ={ - "/exports/home" = { - device = "/home"; - options = [ "bind" ]; - }; - "/stokes" = { - device = "/home"; - options = [ "bind" ]; - }; - "/opt" = { - device = "10.1.63.80:/opt"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" ]; - }; - "/data" = { - device = "10.1.63.80:/data"; - fsType = "nfs"; - options = [ "soft" "rdma" "defaults" ]; - }; - "/vol/local-storage/vol1" = { - device = "/vol/vol1"; - options = [ "bind" ]; - }; - "/vol/local-storage/vol2" = { - device = "/vol/vol2"; - options = [ "bind" ]; - }; - }; - - nix.extraOptions = '' - secret-key-files = /etc/nix/stokes.private - ''; - - services.xserver = { - enable = true; - enableCtrlAltBackspace = true; - layout = "us"; - xkbVariant = "altgr-intl"; - xkbOptions = "eurosign:e"; - displayManager = { - gdm.enable = true; - job.logToFile = true; - }; - desktopManager.xfce.enable = true; - }; - - services.prometheus.alertmanager.configuration.global = { - smtp_smarthost = "smtpgw.itpartner.no:465"; - smtp_auth_username = "utvikling"; - smtp_auth_password = "S0m3rp0m@de#21!"; - smtp_hello = "stokes.regnekraft.io"; - smtp_from = "noreply@stokes.regnekraft.io"; - }; - - services.nginx = { - virtualHosts = { - "ds.matnoc.regnekraft.io" = { - forceSSL = true; - enableACME = true; - serverAliases = []; - locations."/" = { - proxyPass = "http://localhost:9088"; - proxyWebsockets = false; - extraConfig = '' - allow 10.1.2.0/24; - allow 172.19.254.0/24; - allow 172.19.255.0/24; - deny all; - ''; - }; - }; - }; - }; - - # services.gitlab-runner = { - # enable = true; - # extraPackages = with pkgs; [ - # singularity - # ]; - # concurrent = 4; - # services = { - # sif = { - # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; - # executor = "shell"; - # tagList = [ "stokes" "sif" ]; - # }; - # }; - # }; - - # security.sudo.extraConfig = '' - # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity - # ''; - - security.pam = { - services.sshd.googleAuthenticator.enable = true; - loginLimits = [ - { - domain = "@users"; - item = "rss"; - type = "hard"; - value = 16000000; - } - { - domain = "@users"; - item = "cpu"; - type = "hard"; - value = 180; - } - ]; - }; - - # ssh-rsa is deprecated, but putty/winscp users use it - services.openssh.extraConfig = '' - pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 - ''; - - imports = [ ./cluster.nix ./hw/frontend.nix ]; + stokes = import ./frontend.nix { + inherit pkgs; + inherit nodes; }; compute = { @@ -303,7 +60,7 @@ let inherit host; os.externalInterface = "eno33"; hpc.compute = true; - k8s = { inherit etcdCluster; }; + # k8s = { inherit etcdCluster; }; }; node = { @@ -325,7 +82,7 @@ let } ]; }; - interfaces.ibp65s0.2222 = { + interfaces."ibp65s0.2222" = { useDHCP = false; ipv4.addresses = [ { address = ipoib; diff --git a/clusters/stokes/frontend.nix b/clusters/stokes/frontend.nix new file mode 100644 index 0000000..471f305 --- /dev/null +++ b/clusters/stokes/frontend.nix @@ -0,0 +1,235 @@ +{ pkgs, nodes, ...}: +{ + # deployment.tags = [ "frontend" ]; + node.myvnc = true; + + systemd.targets = { + sleep.enable = false; + suspend.enable = false; + hibernate.enable = false; + hybrid-sleep.enable = false; + }; + + features = { + host = { + address = "10.1.62.2"; + name = "c0-0"; + }; + + os = { + externalInterface = "eno1"; + nfs.enable = true; + nfs.exports = '' + /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) + ''; + }; + + hpc = { + slurm.server = true; + frontend = true; + }; + + k8s = { + master.enable = true; + node.enable = true; + inherit nodes; + }; + + monitoring = { + server = { + enable = false; + scrapeHosts = [ "frontend" "mds0-0" ] ++ (builtins.map (x: x.name) nodes); + defaultAlertReceiver = { + email_configs = [ + { to = "jonas.juselius@tromso.serit.no"; } + ]; + }; + pageAlertReceiver = { + webhook_configs = [ + { + url = "https://prometheus-msteams.k2.itpartner.no/stokes"; + http_config = { + tls_config = { insecure_skip_verify = true; }; + }; + } + ]; + }; + }; + webUI.enable = false; + webUI.acmeEmail = "innovasjon@itpartner.no"; + webUI.allow = [ + "10.1.2.0/24" + "172.19.254.0/24" + "172.19.255.0/24" + ]; + infiniband-exporter = { + enable = true; + nameMap = '' + 0x0c42a10300ddc4bc "frontend" + 0x1c34da0300787798 "mds0-0" + 0x0c42a10300dbe7f4 "c0-1" + 0x0c42a10300dbe7d8 "c0-2" + 0x0c42a10300dbe800 "c0-3" + 0x0c42a10300dbec80 "c0-4" + 0x0c42a10300dbea50 "c0-5" + 0x0c42a10300dbeb2c "c0-6" + 0x0c42a10300dbe7fc "c0-7" + 0x0c42a10300dbe5a0 "c0-8" + ''; + }; + slurm-exporter = { + enable = true; + port = 6080; + }; + }; + }; + + networking = { + useDHCP = false; + interfaces.eno1 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.62.2"; + prefixLength = 24; + } ]; + }; + interfaces.enp175s0f0 = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.61.100"; + prefixLength = 24; + } ]; + }; + interfaces."ibp59s0.2222" = { + useDHCP = false; + ipv4.addresses = [ { + address = "10.1.63.100"; + prefixLength = 24; + } ]; + }; + defaultGateway = "10.1.62.1"; + firewall.extraCommands = '' + iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT + iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE + ''; + }; + + fileSystems ={ + "/exports/home" = { + device = "/home"; + options = [ "bind" ]; + }; + "/stokes" = { + device = "/home"; + options = [ "bind" ]; + }; + "/opt" = { + device = "10.1.63.80:/opt"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" ]; + }; + "/data" = { + device = "10.1.63.80:/data"; + fsType = "nfs"; + options = [ "soft" "rdma" "defaults" ]; + }; + "/vol/local-storage/vol1" = { + device = "/vol/vol1"; + options = [ "bind" ]; + }; + "/vol/local-storage/vol2" = { + device = "/vol/vol2"; + options = [ "bind" ]; + }; + }; + + nix.extraOptions = '' + secret-key-files = /etc/nix/stokes.private + ''; + + services.xserver = { + enable = true; + enableCtrlAltBackspace = true; + layout = "us"; + xkbVariant = "altgr-intl"; + xkbOptions = "eurosign:e"; + displayManager = { + gdm.enable = true; + job.logToFile = true; + }; + desktopManager.xfce.enable = true; + }; + + services.prometheus.alertmanager.configuration.global = { + smtp_smarthost = "smtpgw.itpartner.no:465"; + smtp_auth_username = "utvikling"; + smtp_auth_password = "S0m3rp0m@de#21!"; + smtp_hello = "stokes.regnekraft.io"; + smtp_from = "noreply@stokes.regnekraft.io"; + }; + + services.nginx = { + virtualHosts = { + "ds.matnoc.regnekraft.io" = { + forceSSL = true; + enableACME = true; + serverAliases = []; + locations."/" = { + proxyPass = "http://localhost:9088"; + proxyWebsockets = false; + extraConfig = '' + allow 10.1.2.0/24; + allow 172.19.254.0/24; + allow 172.19.255.0/24; + deny all; + ''; + }; + }; + }; + }; + + # services.gitlab-runner = { + # enable = true; + # extraPackages = with pkgs; [ + # singularity + # ]; + # concurrent = 4; + # services = { + # sif = { + # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; + # executor = "shell"; + # tagList = [ "stokes" "sif" ]; + # }; + # }; + # }; + + # security.sudo.extraConfig = '' + # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity + # ''; + + security.pam = { + services.sshd.googleAuthenticator.enable = true; + loginLimits = [ + { + domain = "@users"; + item = "rss"; + type = "hard"; + value = 16000000; + } + { + domain = "@users"; + item = "cpu"; + type = "hard"; + value = 180; + } + ]; + }; + + # ssh-rsa is deprecated, but putty/winscp users use it + services.openssh.extraConfig = '' + pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256 + ''; + + imports = [ ./cluster.nix ./hw/frontend.nix ]; + } diff --git a/clusters/stokes/users.nix b/clusters/stokes/users.nix index 476a6cc..294d32f 100644 --- a/clusters/stokes/users.nix +++ b/clusters/stokes/users.nix @@ -345,6 +345,7 @@ openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuShthrciA4hz3g/e4Q2S1L8OOyb0BpXg3FdjAhBHq8ZWK3GD40qyjmvk45EMX+2hZBXKKjEgO2ToTnH6P0NggwfBU9XhEb/SVxJwohJs55xRERT8jTXcYAXGrZntUg79ndWUHL2NzMMSJnJPEX1M3GZIymDxmUzsaagNvRI3kja42FNHtdX49hGSSygRoqjE+ui2lbFVi6+uY8TUdeW03+BYOgOJ8AtbvwP8MDZqUbHWc7fbg1DE3n52i+Uje2xyXPRwgCKZ0Ha0OLwiezKkVlUqc2gzSIQlKZ2Oy+9AE1knbCr5LVsarERUc17ux74fNQF8P6mCbbqvsgpX0KJK4yrjXPvkFVLcqmRXG+wyYuLLIAuNSG9N6rDgelBevTIqH+zZusXJm2du7mATgpEKBjYHlyS4tuf+gJaP26A2E1Eay5xxUKawm/PY71g/nMHlifYlnbz7fexQ/ObKCntLC0PP07xA6X8einCO81Q+8y0upa4hyzMkfHN4hcknXvj0= doppler@AKVA9454" ]; + }; mib = { description = "Michael Bedington"; @@ -363,5 +364,4 @@ }; # @usr@ }; - }