diff --git a/clusters/stokes/cluster.nix b/clusters/stokes/cluster.nix index b4fe7f6..98ff163 100644 --- a/clusters/stokes/cluster.nix +++ b/clusters/stokes/cluster.nix @@ -137,6 +137,7 @@ let nix = { maxJobs = 32; + trustedUsers = [ "@wheel" ]; # binaryCachePublicKeys = [ # "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig==" # ]; diff --git a/clusters/stokes/default.nix b/clusters/stokes/default.nix index 69805a7..9791ca8 100644 --- a/clusters/stokes/default.nix +++ b/clusters/stokes/default.nix @@ -1,8 +1,8 @@ let # Pin the deployment package-set to a specific version of nixpkgs # pkgs = import (builtins.fetchTarball { - # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; - # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; + # url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz"; + # sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l"; # }) {}; pkgs = import {}; @@ -24,6 +24,7 @@ let map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes; stokes = { + # deployment.tags = [ "frontend" ]; node.myvnc = true; systemd.targets = { @@ -162,8 +163,6 @@ let }; }; - security.pam.services.sshd.googleAuthenticator.enable = true; - nix.extraOptions = '' secret-key-files = /etc/nix/stokes.private ''; @@ -209,10 +208,49 @@ let }; }; + # services.gitlab-runner = { + # enable = true; + # extraPackages = with pkgs; [ + # singularity + # ]; + # concurrent = 4; + # services = { + # sif = { + # registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration"; + # executor = "shell"; + # tagList = [ "stokes" "sif" ]; + # }; + # }; + # }; + + # security.sudo.extraConfig = '' + # gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity + # ''; + + security.pam = { + services.sshd.googleAuthenticator.enable = true; + loginLimits = [ + { + domain = "@users"; + item = "rss"; + type = "hard"; + value = 16000000; + } + { + domain = "@users"; + item = "cpu"; + type = "hard"; + value = 180; + } + ]; + }; + imports = [ ./cluster.nix ./hw/frontend.nix ]; }; compute = { + # deployment.tags = [ "compute" ]; + fileSystems = { "/home/stokes" = { device = "10.1.63.100:/home"; @@ -273,6 +311,16 @@ let } // compute; }; -in - { inherit stokes; } // builtins.foldl' (a: n: a // mkCompute n) {} nodes +in { + ## morph + # network = { + # inherit pkgs; + # description = "stokes"; + # ordering = { + # tags = [ "frontend" "compute" ]; + # }; + # }; + + inherit stokes; +} // builtins.foldl' (a: n: a // mkCompute n) {} nodes diff --git a/clusters/stokes/morph.nix b/clusters/stokes/morph.nix deleted file mode 100644 index c4e1558..0000000 --- a/clusters/stokes/morph.nix +++ /dev/null @@ -1,319 +0,0 @@ -let - # Pin the deployment package-set to a specific version of nixpkgs - # pkgs = import (builtins.fetchTarball { - # url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz"; - # sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36"; - # }) {}; - pkgs = import {}; - - etcdNodes = { - # hpc0-0 = "10.1.63.100"; - # hpc0-1 = "10.1.63.101"; - # hpc0-2 = "10.1.63.102"; - }; - - etcdCluster = { - enable = false; - existing = false; - nodes = etcdNodes; - }; - - k8sNodes = [ - # { name = "hpc0-1"; address = "10.1.61.101"; } - ]; - - stokes = { - deployment.tags = [ "frontend" ]; - node.myvnc = true; - - systemd.targets = { - sleep.enable = false; - suspend.enable = false; - hibernate.enable = false; - hybrid-sleep.enable = false; - }; - - features = { - host = { - address = "10.1.61.100"; - name = "hpc0-0"; - }; - os = { - externalInterface = "eno1"; - nfs.enable = true; - nfs.exports = '' - /exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - /exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash) - ''; - }; - - hpc = { - slurm.server = true; - frontend = true; - }; - - k8s = { - enable = true; - master.enable = true; - node.enable = true; - nodes = nodes; - clusterName = "hpc0"; - initca = ./ca; - cidr = "10.100.0.0/16"; - master = { - name = "hpc0-0"; - address = "10.1.63.100"; - extraSANs = [ "stokes.regnekraft.io" ]; - }; - ingressNodes = [ - "hpc0-0.itpartner.intern" - ]; - fileserver = "mds0-0"; - charts = { - acme_email = "innovasjon@itpartner.no"; - grafana_smtp_user = "utvikling"; - grafana_smtp_password = "S0m3rp0m@de#21!"; - }; - }; - - monitoring = { - server = { - enable = true; - scrapeHosts = [ - "frontend" "mds0-0" - "c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8" - ]; - defaultAlertReceiver = { - email_configs = [ - { to = "jonas.juselius@tromso.serit.no"; } - ]; - }; - pageAlertReceiver = { - webhook_configs = [ - { - url = "https://prometheus-msteams.k2.itpartner.no/stokes"; - http_config = { - tls_config = { insecure_skip_verify = true; }; - }; - } - ]; - }; - }; - webUI.enable = true; - webUI.acmeEmail = "innovasjon@itpartner.no"; - webUI.allow = [ - "10.1.2.0/24" - "172.19.254.0/24" - "172.19.255.0/24" - ]; - infiniband-exporter = { - enable = true; - nameMap = '' - 0x0c42a10300ddc4bc "frontend" - 0x1c34da0300787798 "mds0-0" - 0x0c42a10300dbe7f4 "c0-1" - 0x0c42a10300dbe7d8 "c0-2" - 0x0c42a10300dbe800 "c0-3" - 0x0c42a10300dbec80 "c0-4" - 0x0c42a10300dbea50 "c0-5" - 0x0c42a10300dbeb2c "c0-6" - 0x0c42a10300dbe7fc "c0-7" - 0x0c42a10300dbe5a0 "c0-8" - ''; - }; - slurm-exporter = { - enable = true; - port = 6080; - }; - }; - }; - - networking = { - useDHCP = false; - interfaces.eno1 = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.1.62.2"; - prefixLength = 24; - } ]; - }; - interfaces.enp175s0f0 = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.1.61.100"; - prefixLength = 24; - } ]; - }; - interfaces.ibp59s0 = { - useDHCP = false; - ipv4.addresses = [ { - address = "10.1.63.100"; - prefixLength = 24; - } ]; - }; - defaultGateway = "10.1.62.1"; - firewall.extraCommands = '' - iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT - iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE - ''; - }; - - fileSystems ={ - "/exports/home" = { - device = "/home"; - options = [ "bind" ]; - }; - "/exports/opt" = { - device = "/opt"; - options = [ "bind" ]; - }; - "/data" = { - device = "10.1.63.80:/data"; - fsType = "nfs"; - }; - }; - - security.pam.services.sshd.googleAuthenticator.enable = true; - - nix.extraOptions = '' - secret-key-files = /etc/nix/stokes.private - ''; - - services.xserver = { - enable = true; - enableCtrlAltBackspace = true; - layout = "us"; - xkbVariant = "altgr-intl"; - xkbOptions = "eurosign:e"; - displayManager = { - gdm.enable = true; - job.logToFile = true; - }; - desktopManager.xfce.enable = true; - }; - - services.prometheus.alertmanager.configuration.global = { - smtp_smarthost = "smtpgw.itpartner.no:465"; - smtp_auth_username = "utvikling"; - smtp_auth_password = "S0m3rp0m@de#21!"; - smtp_hello = "stokes.regnekraft.io"; - smtp_from = "noreply@stokes.regnekraft.io"; - }; - - services.nginx = { - virtualHosts = { - "ds.matnoc.regnekraft.io" = { - forceSSL = true; - enableACME = true; - serverAliases = []; - locations."/" = { - proxyPass = "http://localhost:9088"; - proxyWebsockets = false; - extraConfig = '' - allow 10.1.2.0/24; - allow 172.19.254.0/24; - allow 172.19.255.0/24; - deny all; - ''; - }; - }; - }; - }; - - services.minio = { - enable = true; - region = "hpc"; - browser = true; - accessKey = "admin"; - secretKey = "en to tre fire"; - listenAddress = "0.0.0.0:9000"; - dataDir = [ "/data/minio" ]; - }; - - imports = [ ./cluster.nix ./hw/frontend.nix ]; - }; - - compute = { - deployment.tags = [ "compute" ]; - - features = { - os.externalInterface = "eno33"; - hpc.compute = true; - }; - - fileSystems = { - "/home/stokes" = { - device = "10.1.63.100:/home"; - fsType = "nfs"; - }; - "/opt" = { - device = "10.1.63.100:/opt"; - fsType = "nfs"; - }; - "/data" = { - device = "10.1.63.80:/data"; - fsType = "nfs"; - }; - }; - }; - - genComputeNodes = idx: nNodes: - let - nodeList = builtins.genList (x: x + 1) nNodes; - mkCompute = n: - let - ip = "10.1.61.${toString (n + 100)}"; - ipoib = "10.1.63.${toString (n + 100)}"; - name = "c${toString idx}-${toString n}"; - k8sName = "hpc${toString idx}-${toString n}"; - hw = ./hw + "/${name}.nix"; - in { - "${name}" = { - node = { - i40efix = true; - }; - features.host = { - address = ip; - name = k8sName; - }; - networking = { - useDHCP = false; - interfaces.eno33 = { - useDHCP = false; - ipv4.addresses = [ { - address = ip; - prefixLength = 24; - } ]; - ipv4.routes = [ { - address = "10.1.62.2"; - prefixLength = 32; - via = "10.1.61.100"; - } ]; - - }; - interfaces.ibp65s0 = { - useDHCP = false; - ipv4.addresses = [ { - address = ipoib; - prefixLength = 24; - } ]; - }; - }; - imports = [ ./cluster.nix hw ]; - } // compute; - }; - in - builtins.foldl' (a: n: a // mkCompute n) {} nodeList; -in -{ - network = { - inherit pkgs; - description = "stokes"; - ordering = { - tags = [ "frontend" "compute" ]; - }; - }; - - inherit stokes; -} // genComputeNodes 0 8 - diff --git a/clusters/stokes/users.nix b/clusters/stokes/users.nix index dd10a80..91876e1 100644 --- a/clusters/stokes/users.nix +++ b/clusters/stokes/users.nix @@ -18,6 +18,7 @@ yugaos = { gid = 1013; }; ata = { gid = 1014; }; kvile ={ gid = 1015; }; + achim ={ gid = 1016; }; # @grp@ sif = { @@ -37,6 +38,7 @@ "qin" "yugaos" "ata" + "achim" ]; }; @@ -94,6 +96,7 @@ createHome = false; openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAlfc2r3mNkvmdta+H/5zfdFe6317zmCdhhPYbipaGVFPUZO2cCTgSso28oDvOpCDldo/wl3jUxYNDlwH8LYMqKT3aGaOZr8JbxYzd+L+5GM2KTD+4YRmPtpYS/LWcc3j+fiFXSgX6Mrrgf6ineCRuBxSooDVE+pBakM1U7d5NE25apaAvclzFTmZBg0Sf9e5sgHkR99r9DUeGEQWGNZVUGwti39dFVp+aC9dsA+1/OtNB/HMF5G1MMk9dqvN7n7i9o9Plef2DParn4QU1GhmUKeEiBe4OAmSP+WwD4YvK6iXSKZG6tuTEspw+mR3rK5gBHrEiaNlCtp7O9BnAw4Wjhw== rsa-key-20201218" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCp3QEG9uJq4EKMRWlBkqg/EQD6+2E90E6/1i+JLbiBYBSV6Yqac6KzyJezaVXzch+qm6FFXas3thaBScbA4cELkETaxyNzYG6sNcLyEhlQn51pEsq2mFiq8IiaMaDc55/2HTbXVrpNoTNQFt6lBwHziKQYuUI0H0cxze+ppp0ZJOu1MsayW8JOv75YSv6WFIDRR+KP6dOEBu1PsP6plTZwK94ogjQ+3KHGcMAnE3cf8VCEF8akNC6GRmCgXNZE4I9MmKTDr217OoFpnXAx5/KTvGo+USkXc6xn/vbQsni4ExwGlMTg97RK49wIHD1NfGxZ3sv7mZ+UQPqqmSxCG+zueJrR6BSBfbm7fw5KvRn69rOihapeo/6GoqqVDe4yn1imtojjHN6+9pgJ9E6o108qbXRw2X6t1KUuXrB+fTfUKvy0kWiJFIMDSUtKF/nhiES+aCI4b4WBwyg5hdKGvgJdjyUS7P/jYgqWRe+qmknAERtQKlFDA/C6ChsTXerFD5Ikvu3dajJUiDehszEON5F4JlxSf2VpUFCDLVNqV/GjJqOg90mXGDk82c+0ZHIUsPLsdqR+t/xnOSv1Ks9I5fId6g+3OlR1ifnb3Qm48QGKbi/CM8M/QXzv4VgeIkRTR0Oi4W0P1tpUSyPd1nyGaM/B/FqN52XIUjRqIfu0emwgiw== olean@navier" ]; }; @@ -319,6 +322,24 @@ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtB+HWtE4iXJiRVi1MUKaE3R3FAcHzCgiF84ho6GXKxx5H2iY8sgfxWo/lFSonhZKTo/+dHOYNKs42Q85ytG1rpcEYYVOK53mx8f7Z3THmw348a/+geM8Bukvo5pLc7KmXIvq6UQIjZmI/wnbA7B8MzLyrod71SaT1ujMEV1Jg0b3KnjS5kJnUHDICw3CdvuenNIgYl/zbTeEJ1iUu6T1TY+cNGG/7HOsaR1leCArDutHIKowcIFQFZoLEikM2DX5MSp9UBizAVogHugEqE2Bqh+C7NyTzJfQzR8s4drnt9IaptJQmCo6z9f+dQALjhftJXBDdkR6coMyOujV3Yyc5 rsa-key-20210928" ]; }; + + achim = { + description = "Achim Randelhoff"; + home = "/home/achim"; + group = "achim"; + extraGroups = [ + "users" + ]; + uid = 1016; + isNormalUser = true; + createHome = false; + useDefaultShell = true; + # shell = pkgs.fish; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuShthrciA4hz3g/e4Q2S1L8OOyb0BpXg3FdjAhBHq8ZWK3GD40qyjmvk45EMX+2hZBXKKjEgO2ToTnH6P0NggwfBU9XhEb/SVxJwohJs55xRERT8jTXcYAXGrZntUg79ndWUHL2NzMMSJnJPEX1M3GZIymDxmUzsaagNvRI3kja42FNHtdX49hGSSygRoqjE+ui2lbFVi6+uY8TUdeW03+BYOgOJ8AtbvwP8MDZqUbHWc7fbg1DE3n52i+Uje2xyXPRwgCKZ0Ha0OLwiezKkVlUqc2gzSIQlKZ2Oy+9AE1knbCr5LVsarERUc17ux74fNQF8P6mCbbqvsgpX0KJK4yrjXPvkFVLcqmRXG+wyYuLLIAuNSG9N6rDgelBevTIqH+zZusXJm2du7mATgpEKBjYHlyS4tuf+gJaP26A2E1Eay5xxUKawm/PY71g/nMHlifYlnbz7fexQ/ObKCntLC0PP07xA6X8einCO81Q+8y0upa4hyzMkfHN4hcknXvj0= doppler@AKVA9454 +" + ]; + }; # @usr@ };