stokes: misc stuff, new users

clusters/stokes/cluster.nix

@@ -137,6 +137,7 @@ let
 
     nix = {
       maxJobs = 32;
+      trustedUsers = [ "@wheel" ];
       # binaryCachePublicKeys = [
       #   "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig=="
       # ];

clusters/stokes/default.nix

@@ -1,8 +1,8 @@
 let
   # Pin the deployment package-set to a specific version of nixpkgs
   # pkgs = import (builtins.fetchTarball {
-  #   url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
+  #   url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
-  #   sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
+  #   sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
   # }) {};
   pkgs = import <nixpkgs> {};
 
@@ -24,6 +24,7 @@ let
     map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;
 
   stokes = {
+    # deployment.tags = [ "frontend" ];
     node.myvnc = true;
 
     systemd.targets = {
@@ -162,8 +163,6 @@ let
       };
     };
 
-    security.pam.services.sshd.googleAuthenticator.enable = true;
-
     nix.extraOptions = ''
       secret-key-files = /etc/nix/stokes.private
     '';
@@ -209,10 +208,49 @@ let
       };
     };
 
+    # services.gitlab-runner = {
+    #   enable = true;
+    #   extraPackages = with pkgs; [
+    #     singularity
+    #   ];
+    #   concurrent = 4;
+    #   services = {
+    #     sif = {
+    #       registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
+    #       executor = "shell";
+    #       tagList = [ "stokes" "sif" ];
+    #     };
+    #   };
+    # };
+
+    # security.sudo.extraConfig = ''
+    #   gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
+    # '';
+
+    security.pam = {
+      services.sshd.googleAuthenticator.enable = true;
+      loginLimits = [
+        {
+          domain = "@users";
+          item = "rss";
+          type = "hard";
+          value = 16000000;
+        }
+        {
+          domain = "@users";
+          item = "cpu";
+          type = "hard";
+          value = 180;
+        }
+      ];
+    };
+
     imports = [ ./cluster.nix ./hw/frontend.nix ];
   };
 
   compute = {
+    # deployment.tags = [ "compute" ];
+
     fileSystems = {
       "/home/stokes" = {
         device = "10.1.63.100:/home";
@@ -273,6 +311,16 @@ let
     }
     // compute;
 };
-in
+in {
-  { inherit stokes; } // builtins.foldl' (a: n: a // mkCompute n) {} nodes
+    ## morph
+    # network =  {
+    #   inherit pkgs;
+    #   description = "stokes";
+    #   ordering = {
+    #     tags = [ "frontend" "compute" ];
+    #   };
+    # };
+
+    inherit stokes;
+} // builtins.foldl' (a: n: a // mkCompute n) {} nodes
 

clusters/stokes/morph.nix

@@ -1,319 +0,0 @@
-let
-  # Pin the deployment package-set to a specific version of nixpkgs
-  # pkgs = import (builtins.fetchTarball {
-  #   url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
-  #   sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
-  # }) {};
-  pkgs = import <nixpkgs> {};
-
-  etcdNodes = {
-    # hpc0-0 = "10.1.63.100";
-    # hpc0-1 = "10.1.63.101";
-    # hpc0-2 = "10.1.63.102";
-  };
-
-  etcdCluster = {
-    enable = false;
-    existing = false;
-    nodes = etcdNodes;
-  };
-
-  k8sNodes = [
-    # { name = "hpc0-1"; address = "10.1.61.101"; }
-  ];
-
-  stokes = {
-    deployment.tags = [ "frontend" ];
-    node.myvnc = true;
-
-    systemd.targets = {
-      sleep.enable = false;
-      suspend.enable = false;
-      hibernate.enable = false;
-      hybrid-sleep.enable = false;
-    };
-
-    features = {
-      host = {
-        address = "10.1.61.100";
-        name = "hpc0-0";
-      };
-      os = {
-        externalInterface = "eno1";
-        nfs.enable = true;
-        nfs.exports = ''
-          /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-          /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
-        '';
-      };
-
-      hpc = {
-        slurm.server = true;
-        frontend = true;
-      };
-
-      k8s = {
-        enable = true;
-        master.enable = true;
-        node.enable = true;
-        nodes = nodes;
-        clusterName = "hpc0";
-        initca = ./ca;
-        cidr = "10.100.0.0/16";
-        master = {
-          name = "hpc0-0";
-          address = "10.1.63.100";
-          extraSANs = [ "stokes.regnekraft.io" ];
-        };
-        ingressNodes = [
-          "hpc0-0.itpartner.intern"
-        ];
-        fileserver = "mds0-0";
-        charts = {
-          acme_email = "innovasjon@itpartner.no";
-          grafana_smtp_user = "utvikling";
-          grafana_smtp_password = "S0m3rp0m@de#21!";
-        };
-      };
-
-      monitoring = {
-        server = {
-          enable = true;
-          scrapeHosts = [
-            "frontend" "mds0-0"
-            "c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8"
-          ];
-          defaultAlertReceiver = {
-            email_configs = [
-              { to = "jonas.juselius@tromso.serit.no"; }
-            ];
-          };
-          pageAlertReceiver = {
-            webhook_configs = [
-              {
-                url = "https://prometheus-msteams.k2.itpartner.no/stokes";
-                http_config = {
-                  tls_config = { insecure_skip_verify = true; };
-                };
-              }
-            ];
-          };
-        };
-        webUI.enable = true;
-        webUI.acmeEmail = "innovasjon@itpartner.no";
-        webUI.allow = [
-            "10.1.2.0/24"
-            "172.19.254.0/24"
-            "172.19.255.0/24"
-        ];
-        infiniband-exporter = {
-          enable = true;
-          nameMap = ''
-            0x0c42a10300ddc4bc "frontend"
-            0x1c34da0300787798 "mds0-0"
-            0x0c42a10300dbe7f4 "c0-1"
-            0x0c42a10300dbe7d8 "c0-2"
-            0x0c42a10300dbe800 "c0-3"
-            0x0c42a10300dbec80 "c0-4"
-            0x0c42a10300dbea50 "c0-5"
-            0x0c42a10300dbeb2c "c0-6"
-            0x0c42a10300dbe7fc "c0-7"
-            0x0c42a10300dbe5a0 "c0-8"
-          '';
-        };
-        slurm-exporter = {
-          enable = true;
-          port = 6080;
-        };
-      };
-    };
-
-    networking = {
-      useDHCP = false;
-      interfaces.eno1 = {
-        useDHCP = false;
-        ipv4.addresses = [ {
-          address = "10.1.62.2";
-          prefixLength = 24;
-        } ];
-      };
-      interfaces.enp175s0f0 = {
-        useDHCP = false;
-        ipv4.addresses = [ {
-          address = "10.1.61.100";
-          prefixLength = 24;
-        } ];
-      };
-      interfaces.ibp59s0 = {
-        useDHCP = false;
-        ipv4.addresses = [ {
-          address = "10.1.63.100";
-          prefixLength = 24;
-        } ];
-      };
-      defaultGateway = "10.1.62.1";
-      firewall.extraCommands = ''
-        iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT
-        iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE
-      '';
-    };
-
-    fileSystems ={
-      "/exports/home" = {
-        device = "/home";
-        options = [ "bind" ];
-      };
-      "/exports/opt" = {
-        device = "/opt";
-        options = [ "bind" ];
-      };
-      "/data" = {
-        device = "10.1.63.80:/data";
-        fsType = "nfs";
-      };
-    };
-
-    security.pam.services.sshd.googleAuthenticator.enable = true;
-
-    nix.extraOptions = ''
-      secret-key-files = /etc/nix/stokes.private
-    '';
-
-    services.xserver = {
-      enable = true;
-      enableCtrlAltBackspace = true;
-      layout = "us";
-      xkbVariant = "altgr-intl";
-      xkbOptions = "eurosign:e";
-      displayManager = {
-        gdm.enable = true;
-        job.logToFile = true;
-      };
-      desktopManager.xfce.enable = true;
-    };
-
-    services.prometheus.alertmanager.configuration.global = {
-        smtp_smarthost = "smtpgw.itpartner.no:465";
-        smtp_auth_username = "utvikling";
-        smtp_auth_password = "S0m3rp0m@de#21!";
-        smtp_hello = "stokes.regnekraft.io";
-        smtp_from = "noreply@stokes.regnekraft.io";
-    };
-
-    services.nginx = {
-      virtualHosts = {
-        "ds.matnoc.regnekraft.io" = {
-          forceSSL = true;
-          enableACME = true;
-          serverAliases = [];
-          locations."/" = {
-            proxyPass = "http://localhost:9088";
-            proxyWebsockets = false;
-            extraConfig = ''
-              allow 10.1.2.0/24;
-              allow 172.19.254.0/24;
-              allow 172.19.255.0/24;
-              deny all;
-            '';
-          };
-        };
-      };
-    };
-
-    services.minio = {
-      enable = true;
-      region = "hpc";
-      browser = true;
-      accessKey = "admin";
-      secretKey = "en to tre fire";
-      listenAddress = "0.0.0.0:9000";
-      dataDir = [ "/data/minio" ];
-    };
-
-    imports = [ ./cluster.nix ./hw/frontend.nix ];
-  };
-
-  compute = {
-    deployment.tags = [ "compute" ];
-
-    features = {
-      os.externalInterface = "eno33";
-      hpc.compute = true;
-    };
-
-    fileSystems = {
-      "/home/stokes" = {
-        device = "10.1.63.100:/home";
-        fsType = "nfs";
-      };
-      "/opt" = {
-        device = "10.1.63.100:/opt";
-        fsType = "nfs";
-      };
-      "/data" = {
-        device = "10.1.63.80:/data";
-        fsType = "nfs";
-      };
-    };
-  };
-
-  genComputeNodes = idx: nNodes:
-    let
-      nodeList = builtins.genList (x: x + 1) nNodes;
-      mkCompute = n:
-        let
-           ip    = "10.1.61.${toString (n + 100)}";
-           ipoib = "10.1.63.${toString (n + 100)}";
-           name = "c${toString idx}-${toString n}";
-           k8sName = "hpc${toString idx}-${toString n}";
-           hw = ./hw + "/${name}.nix";
-        in {
-          "${name}" = {
-            node = {
-              i40efix = true;
-            };
-            features.host = {
-              address = ip;
-              name = k8sName;
-            };
-            networking = {
-              useDHCP = false;
-              interfaces.eno33 = {
-                useDHCP = false;
-                ipv4.addresses = [ {
-                  address = ip;
-                  prefixLength = 24;
-                } ];
-                ipv4.routes = [ {
-                  address = "10.1.62.2";
-                  prefixLength = 32;
-                  via = "10.1.61.100";
-                } ];
-
-              };
-              interfaces.ibp65s0 = {
-                useDHCP = false;
-                ipv4.addresses = [ {
-                  address = ipoib;
-                  prefixLength = 24;
-                } ];
-              };
-            };
-            imports = [ ./cluster.nix hw ];
-          } // compute;
-        };
-    in
-      builtins.foldl' (a: n: a // mkCompute n) {} nodeList;
-in
-{
-  network =  {
-    inherit pkgs;
-    description = "stokes";
-    ordering = {
-      tags = [ "frontend" "compute" ];
-    };
-  };
-
-  inherit stokes;
-} // genComputeNodes 0 8
-

clusters/stokes/users.nix

@@ -18,6 +18,7 @@
     yugaos = { gid = 1013; };
     ata = { gid = 1014; };
     kvile ={ gid = 1015; };
+    achim ={ gid = 1016; };
     # @grp@
 
     sif = {
@@ -37,6 +38,7 @@
         "qin"
         "yugaos"
         "ata"
+        "achim"
       ];
     };
 
@@ -94,6 +96,7 @@
       createHome = false;
       openssh.authorizedKeys.keys = [
         "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAlfc2r3mNkvmdta+H/5zfdFe6317zmCdhhPYbipaGVFPUZO2cCTgSso28oDvOpCDldo/wl3jUxYNDlwH8LYMqKT3aGaOZr8JbxYzd+L+5GM2KTD+4YRmPtpYS/LWcc3j+fiFXSgX6Mrrgf6ineCRuBxSooDVE+pBakM1U7d5NE25apaAvclzFTmZBg0Sf9e5sgHkR99r9DUeGEQWGNZVUGwti39dFVp+aC9dsA+1/OtNB/HMF5G1MMk9dqvN7n7i9o9Plef2DParn4QU1GhmUKeEiBe4OAmSP+WwD4YvK6iXSKZG6tuTEspw+mR3rK5gBHrEiaNlCtp7O9BnAw4Wjhw== rsa-key-20201218"
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCp3QEG9uJq4EKMRWlBkqg/EQD6+2E90E6/1i+JLbiBYBSV6Yqac6KzyJezaVXzch+qm6FFXas3thaBScbA4cELkETaxyNzYG6sNcLyEhlQn51pEsq2mFiq8IiaMaDc55/2HTbXVrpNoTNQFt6lBwHziKQYuUI0H0cxze+ppp0ZJOu1MsayW8JOv75YSv6WFIDRR+KP6dOEBu1PsP6plTZwK94ogjQ+3KHGcMAnE3cf8VCEF8akNC6GRmCgXNZE4I9MmKTDr217OoFpnXAx5/KTvGo+USkXc6xn/vbQsni4ExwGlMTg97RK49wIHD1NfGxZ3sv7mZ+UQPqqmSxCG+zueJrR6BSBfbm7fw5KvRn69rOihapeo/6GoqqVDe4yn1imtojjHN6+9pgJ9E6o108qbXRw2X6t1KUuXrB+fTfUKvy0kWiJFIMDSUtKF/nhiES+aCI4b4WBwyg5hdKGvgJdjyUS7P/jYgqWRe+qmknAERtQKlFDA/C6ChsTXerFD5Ikvu3dajJUiDehszEON5F4JlxSf2VpUFCDLVNqV/GjJqOg90mXGDk82c+0ZHIUsPLsdqR+t/xnOSv1Ks9I5fId6g+3OlR1ifnb3Qm48QGKbi/CM8M/QXzv4VgeIkRTR0Oi4W0P1tpUSyPd1nyGaM/B/FqN52XIUjRqIfu0emwgiw== olean@navier"
       ];
     };
 
@@ -319,6 +322,24 @@
         "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtB+HWtE4iXJiRVi1MUKaE3R3FAcHzCgiF84ho6GXKxx5H2iY8sgfxWo/lFSonhZKTo/+dHOYNKs42Q85ytG1rpcEYYVOK53mx8f7Z3THmw348a/+geM8Bukvo5pLc7KmXIvq6UQIjZmI/wnbA7B8MzLyrod71SaT1ujMEV1Jg0b3KnjS5kJnUHDICw3CdvuenNIgYl/zbTeEJ1iUu6T1TY+cNGG/7HOsaR1leCArDutHIKowcIFQFZoLEikM2DX5MSp9UBizAVogHugEqE2Bqh+C7NyTzJfQzR8s4drnt9IaptJQmCo6z9f+dQALjhftJXBDdkR6coMyOujV3Yyc5 rsa-key-20210928"
       ];
     };
+
+    achim = {
+      description = "Achim Randelhoff";
+      home = "/home/achim";
+      group = "achim";
+      extraGroups = [
+        "users"
+      ];
+      uid = 1016;
+      isNormalUser = true;
+      createHome = false;
+      useDefaultShell = true;
+      # shell = pkgs.fish;
+      openssh.authorizedKeys.keys = [
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuShthrciA4hz3g/e4Q2S1L8OOyb0BpXg3FdjAhBHq8ZWK3GD40qyjmvk45EMX+2hZBXKKjEgO2ToTnH6P0NggwfBU9XhEb/SVxJwohJs55xRERT8jTXcYAXGrZntUg79ndWUHL2NzMMSJnJPEX1M3GZIymDxmUzsaagNvRI3kja42FNHtdX49hGSSygRoqjE+ui2lbFVi6+uY8TUdeW03+BYOgOJ8AtbvwP8MDZqUbHWc7fbg1DE3n52i+Uje2xyXPRwgCKZ0Ha0OLwiezKkVlUqc2gzSIQlKZ2Oy+9AE1knbCr5LVsarERUc17ux74fNQF8P6mCbbqvsgpX0KJK4yrjXPvkFVLcqmRXG+wyYuLLIAuNSG9N6rDgelBevTIqH+zZusXJm2du7mATgpEKBjYHlyS4tuf+gJaP26A2E1Eay5xxUKawm/PY71g/nMHlifYlnbz7fexQ/ObKCntLC0PP07xA6X8einCO81Q+8y0upa4hyzMkfHN4hcknXvj0= doppler@AKVA9454
+"
+      ];
+    };
     # @usr@
   };