split frontend into separate module

clusters/stokes/cluster.nix

@@ -3,6 +3,17 @@ with lib;
 let
   cfg = config.features.host;
 
+  etcdCluster = {
+    enable = true;
+    existing = true;
+    nodes =
+      {
+        c0-0 = "10.1.61.100";
+        c0-1 = "10.1.61.101";
+        c0-2 = "10.1.61.102";
+      };
+  };
+
   mkSANs = host: [
     host.name
     host.address
@@ -101,6 +112,7 @@ let
         slurm = {
           client = true;
           mungeKey = ./munge.key;
+          mungeUid = 996; # hack
           controlMachine = "stokes";
           nodeName = [
             "c0-[1-8] Sockets=1 CoresPerSocket=64 ThreadsPerCore=1 RealMemory=256000 TmpDisk=100000 State=UNKNOWN"
@@ -130,6 +142,7 @@ let
         enable = true;
         node.enable = true;
         clusterName = "hpc0";
+        inherit etcdCluster;
         initca = ./ca;
         cidr = "10.100.0.0/16";
         master = {

clusters/stokes/default.nix

@@ -6,257 +6,14 @@ let
   # }) {};
   pkgs = import <nixpkgs> {};
 
-  etcdNodes = {
-    c0-0 = "10.1.61.100";
-    c0-1 = "10.1.61.101";
-    c0-2 = "10.1.61.102";
-  };
-
-  etcdCluster = {
-    enable = true;
-    existing = true;
-    nodes = etcdNodes;
-  };
-
   nodes =
     with builtins;
     let nodes = genList (n: n + 1) 8; in
     map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;
 
-  stokes = {
+  stokes = import ./frontend.nix {
-    # deployment.tags = [ "frontend" ];
+    inherit pkgs;
-    node.myvnc = true;
+    inherit nodes;
-
-    systemd.targets = {
-      sleep.enable = false;
-      suspend.enable = false;
-      hibernate.enable = false;
-      hybrid-sleep.enable = false;
-    };
-
-    features = {
-      host = {
-        address = "10.1.62.2";
-        name = "c0-0";
-      };
-
-      os = {
-        externalInterface = "eno1";
-        nfs.enable = true;
-        nfs.exports = ''
-          /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-          /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-        '';
-      };
-
-      hpc = {
-        slurm.server = true;
-        frontend = true;
-      };
-
-      k8s = {
-        master.enable = true;
-        node.enable = true;
-        inherit nodes;
-        inherit etcdCluster;
-      };
-
-      monitoring = {
-        server = {
-          enable = false;
-          scrapeHosts = [ "frontend" "mds0-0" ] ++ (builtins.map (x: x.name) nodes);
-          defaultAlertReceiver = {
-            email_configs = [
-              { to = "jonas.juselius@tromso.serit.no"; }
-            ];
-          };
-          pageAlertReceiver = {
-            webhook_configs = [
-              {
-                url = "https://prometheus-msteams.k2.itpartner.no/stokes";
-                http_config = {
-                  tls_config = { insecure_skip_verify = true; };
-                };
-              }
-            ];
-          };
-        };
-        webUI.enable = false;
-        webUI.acmeEmail = "innovasjon@itpartner.no";
-        webUI.allow = [
-            "10.1.2.0/24"
-            "172.19.254.0/24"
-            "172.19.255.0/24"
-        ];
-        infiniband-exporter = {
-          enable = true;
-          nameMap = ''
-            0x0c42a10300ddc4bc "frontend"
-            0x1c34da0300787798 "mds0-0"
-            0x0c42a10300dbe7f4 "c0-1"
-            0x0c42a10300dbe7d8 "c0-2"
-            0x0c42a10300dbe800 "c0-3"
-            0x0c42a10300dbec80 "c0-4"
-            0x0c42a10300dbea50 "c0-5"
-            0x0c42a10300dbeb2c "c0-6"
-            0x0c42a10300dbe7fc "c0-7"
-            0x0c42a10300dbe5a0 "c0-8"
-          '';
-        };
-        slurm-exporter = {
-          enable = true;
-          port = 6080;
-        };
-      };
-    };
-
-    networking = {
-      useDHCP = false;
-      interfaces.eno1 = {
-        useDHCP = false;
-        ipv4.addresses = [ {
-          address = "10.1.62.2";
-          prefixLength = 24;
-        } ];
-      };
-      interfaces.enp175s0f0 = {
-        useDHCP = false;
-        ipv4.addresses = [ {
-          address = "10.1.61.100";
-          prefixLength = 24;
-        } ];
-      };
-      interfaces.ibp59s0 = {
-        useDHCP = false;
-        ipv4.addresses = [ {
-          address = "10.1.63.100";
-          prefixLength = 24;
-        } ];
-      };
-      defaultGateway = "10.1.62.1";
-      firewall.extraCommands = ''
-        iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT
-        iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE
-      '';
-    };
-
-    fileSystems ={
-      "/exports/home" = {
-        device = "/home";
-        options = [ "bind" ];
-      };
-      "/stokes" = {
-        device = "/home";
-        options = [ "bind" ];
-      };
-      "/opt" = {
-        device = "10.1.63.80:/opt";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" ];
-      };
-      "/data" = {
-        device = "10.1.63.80:/data";
-        fsType = "nfs";
-        options = [ "soft" "rdma" "defaults" ];
-      };
-      "/vol/local-storage/vol1" = {
-        device = "/vol/vol1";
-        options = [ "bind" ];
-      };
-      "/vol/local-storage/vol2" = {
-        device = "/vol/vol2";
-        options = [ "bind" ];
-      };
-    };
-
-    nix.extraOptions = ''
-      secret-key-files = /etc/nix/stokes.private
-    '';
-
-    services.xserver = {
-      enable = true;
-      enableCtrlAltBackspace = true;
-      layout = "us";
-      xkbVariant = "altgr-intl";
-      xkbOptions = "eurosign:e";
-      displayManager = {
-        gdm.enable = true;
-        job.logToFile = true;
-      };
-      desktopManager.xfce.enable = true;
-    };
-
-    services.prometheus.alertmanager.configuration.global = {
-        smtp_smarthost = "smtpgw.itpartner.no:465";
-        smtp_auth_username = "utvikling";
-        smtp_auth_password = "S0m3rp0m@de#21!";
-        smtp_hello = "stokes.regnekraft.io";
-        smtp_from = "noreply@stokes.regnekraft.io";
-    };
-
-    services.nginx = {
-      virtualHosts = {
-        "ds.matnoc.regnekraft.io" = {
-          forceSSL = true;
-          enableACME = true;
-          serverAliases = [];
-          locations."/" = {
-            proxyPass = "http://localhost:9088";
-            proxyWebsockets = false;
-            extraConfig = ''
-              allow 10.1.2.0/24;
-              allow 172.19.254.0/24;
-              allow 172.19.255.0/24;
-              deny all;
-            '';
-          };
-        };
-      };
-    };
-
-    # services.gitlab-runner = {
-    #   enable = true;
-    #   extraPackages = with pkgs; [
-    #     singularity
-    #   ];
-    #   concurrent = 4;
-    #   services = {
-    #     sif = {
-    #       registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
-    #       executor = "shell";
-    #       tagList = [ "stokes" "sif" ];
-    #     };
-    #   };
-    # };
-
-    # security.sudo.extraConfig = ''
-    #   gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
-    # '';
-
-    security.pam = {
-      services.sshd.googleAuthenticator.enable = true;
-      loginLimits = [
-        {
-          domain = "@users";
-          item = "rss";
-          type = "hard";
-          value = 16000000;
-        }
-        {
-          domain = "@users";
-          item = "cpu";
-          type = "hard";
-          value = 180;
-        }
-      ];
-    };
-
-    # ssh-rsa is deprecated, but putty/winscp users use it
-    services.openssh.extraConfig = ''
-      pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
-    '';
-
-    imports = [ ./cluster.nix ./hw/frontend.nix ];
   };
 
   compute = {
@@ -303,7 +60,7 @@ let
         inherit host;
         os.externalInterface = "eno33";
         hpc.compute = true;
-        k8s = { inherit etcdCluster; };
+        # k8s = { inherit etcdCluster; };
       };
 
       node = {
@@ -325,7 +82,7 @@ let
           } ];
 
         };
-        interfaces.ibp65s0.2222 = {
+        interfaces."ibp65s0.2222" = {
           useDHCP = false;
           ipv4.addresses = [ {
             address = ipoib;

clusters/stokes/frontend.nix

@@ -0,0 +1,235 @@
+{ pkgs, nodes, ...}:
+{
+    # deployment.tags = [ "frontend" ];
+    node.myvnc = true;
+
+    systemd.targets = {
+      sleep.enable = false;
+      suspend.enable = false;
+      hibernate.enable = false;
+      hybrid-sleep.enable = false;
+    };
+
+    features = {
+      host = {
+        address = "10.1.62.2";
+        name = "c0-0";
+      };
+
+      os = {
+        externalInterface = "eno1";
+        nfs.enable = true;
+        nfs.exports = ''
+          /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+          /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
+        '';
+      };
+
+      hpc = {
+        slurm.server = true;
+        frontend = true;
+      };
+
+      k8s = {
+        master.enable = true;
+        node.enable = true;
+        inherit nodes;
+      };
+
+      monitoring = {
+        server = {
+          enable = false;
+          scrapeHosts = [ "frontend" "mds0-0" ] ++ (builtins.map (x: x.name) nodes);
+          defaultAlertReceiver = {
+            email_configs = [
+              { to = "jonas.juselius@tromso.serit.no"; }
+            ];
+          };
+          pageAlertReceiver = {
+            webhook_configs = [
+              {
+                url = "https://prometheus-msteams.k2.itpartner.no/stokes";
+                http_config = {
+                  tls_config = { insecure_skip_verify = true; };
+                };
+              }
+            ];
+          };
+        };
+        webUI.enable = false;
+        webUI.acmeEmail = "innovasjon@itpartner.no";
+        webUI.allow = [
+            "10.1.2.0/24"
+            "172.19.254.0/24"
+            "172.19.255.0/24"
+        ];
+        infiniband-exporter = {
+          enable = true;
+          nameMap = ''
+            0x0c42a10300ddc4bc "frontend"
+            0x1c34da0300787798 "mds0-0"
+            0x0c42a10300dbe7f4 "c0-1"
+            0x0c42a10300dbe7d8 "c0-2"
+            0x0c42a10300dbe800 "c0-3"
+            0x0c42a10300dbec80 "c0-4"
+            0x0c42a10300dbea50 "c0-5"
+            0x0c42a10300dbeb2c "c0-6"
+            0x0c42a10300dbe7fc "c0-7"
+            0x0c42a10300dbe5a0 "c0-8"
+          '';
+        };
+        slurm-exporter = {
+          enable = true;
+          port = 6080;
+        };
+      };
+    };
+
+    networking = {
+      useDHCP = false;
+      interfaces.eno1 = {
+        useDHCP = false;
+        ipv4.addresses = [ {
+          address = "10.1.62.2";
+          prefixLength = 24;
+        } ];
+      };
+      interfaces.enp175s0f0 = {
+        useDHCP = false;
+        ipv4.addresses = [ {
+          address = "10.1.61.100";
+          prefixLength = 24;
+        } ];
+      };
+      interfaces."ibp59s0.2222" = {
+        useDHCP = false;
+        ipv4.addresses = [ {
+          address = "10.1.63.100";
+          prefixLength = 24;
+        } ];
+      };
+      defaultGateway = "10.1.62.1";
+      firewall.extraCommands = ''
+        iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT
+        iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE
+      '';
+    };
+
+    fileSystems ={
+      "/exports/home" = {
+        device = "/home";
+        options = [ "bind" ];
+      };
+      "/stokes" = {
+        device = "/home";
+        options = [ "bind" ];
+      };
+      "/opt" = {
+        device = "10.1.63.80:/opt";
+        fsType = "nfs";
+        options = [ "soft" "rdma" "defaults" ];
+      };
+      "/data" = {
+        device = "10.1.63.80:/data";
+        fsType = "nfs";
+        options = [ "soft" "rdma" "defaults" ];
+      };
+      "/vol/local-storage/vol1" = {
+        device = "/vol/vol1";
+        options = [ "bind" ];
+      };
+      "/vol/local-storage/vol2" = {
+        device = "/vol/vol2";
+        options = [ "bind" ];
+      };
+    };
+
+    nix.extraOptions = ''
+      secret-key-files = /etc/nix/stokes.private
+    '';
+
+    services.xserver = {
+      enable = true;
+      enableCtrlAltBackspace = true;
+      layout = "us";
+      xkbVariant = "altgr-intl";
+      xkbOptions = "eurosign:e";
+      displayManager = {
+        gdm.enable = true;
+        job.logToFile = true;
+      };
+      desktopManager.xfce.enable = true;
+    };
+
+    services.prometheus.alertmanager.configuration.global = {
+        smtp_smarthost = "smtpgw.itpartner.no:465";
+        smtp_auth_username = "utvikling";
+        smtp_auth_password = "S0m3rp0m@de#21!";
+        smtp_hello = "stokes.regnekraft.io";
+        smtp_from = "noreply@stokes.regnekraft.io";
+    };
+
+    services.nginx = {
+      virtualHosts = {
+        "ds.matnoc.regnekraft.io" = {
+          forceSSL = true;
+          enableACME = true;
+          serverAliases = [];
+          locations."/" = {
+            proxyPass = "http://localhost:9088";
+            proxyWebsockets = false;
+            extraConfig = ''
+              allow 10.1.2.0/24;
+              allow 172.19.254.0/24;
+              allow 172.19.255.0/24;
+              deny all;
+            '';
+          };
+        };
+      };
+    };
+
+    # services.gitlab-runner = {
+    #   enable = true;
+    #   extraPackages = with pkgs; [
+    #     singularity
+    #   ];
+    #   concurrent = 4;
+    #   services = {
+    #     sif = {
+    #       registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
+    #       executor = "shell";
+    #       tagList = [ "stokes" "sif" ];
+    #     };
+    #   };
+    # };
+
+    # security.sudo.extraConfig = ''
+    #   gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
+    # '';
+
+    security.pam = {
+      services.sshd.googleAuthenticator.enable = true;
+      loginLimits = [
+        {
+          domain = "@users";
+          item = "rss";
+          type = "hard";
+          value = 16000000;
+        }
+        {
+          domain = "@users";
+          item = "cpu";
+          type = "hard";
+          value = 180;
+        }
+      ];
+    };
+
+    # ssh-rsa is deprecated, but putty/winscp users use it
+    services.openssh.extraConfig = ''
+      pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
+    '';
+
+    imports = [ ./cluster.nix ./hw/frontend.nix ];
+  }

clusters/stokes/users.nix

@@ -345,6 +345,7 @@
       openssh.authorizedKeys.keys = [
         "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuShthrciA4hz3g/e4Q2S1L8OOyb0BpXg3FdjAhBHq8ZWK3GD40qyjmvk45EMX+2hZBXKKjEgO2ToTnH6P0NggwfBU9XhEb/SVxJwohJs55xRERT8jTXcYAXGrZntUg79ndWUHL2NzMMSJnJPEX1M3GZIymDxmUzsaagNvRI3kja42FNHtdX49hGSSygRoqjE+ui2lbFVi6+uY8TUdeW03+BYOgOJ8AtbvwP8MDZqUbHWc7fbg1DE3n52i+Uje2xyXPRwgCKZ0Ha0OLwiezKkVlUqc2gzSIQlKZ2Oy+9AE1knbCr5LVsarERUc17ux74fNQF8P6mCbbqvsgpX0KJK4yrjXPvkFVLcqmRXG+wyYuLLIAuNSG9N6rDgelBevTIqH+zZusXJm2du7mATgpEKBjYHlyS4tuf+gJaP26A2E1Eay5xxUKawm/PY71g/nMHlifYlnbz7fexQ/ObKCntLC0PP07xA6X8einCO81Q+8y0upa4hyzMkfHN4hcknXvj0= doppler@AKVA9454"
       ];
+    };
 
     mib = {
       description = "Michael Bedington";
@@ -363,5 +364,4 @@
     };
     # @usr@
   };
-
 }