oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

stokes: misc stuff, new users

Browse Source
This commit is contained in:
Jonas Juselius
2022-01-24 15:00:52 +01:00
parent 9bf1722b03
commit 67a53eb442
4 changed files with 76 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
clusters/stokes/cluster.nix
View File

@@ -137,6 +137,7 @@ let
nix = {
maxJobs = 32;
trustedUsers = [ "@wheel" ];
# binaryCachePublicKeys = [
# "stokes-1:BCgUFnXc6wgpstwG0M09/Ccrrz45MxHpS62JSC9sxW5hWxMqBNNvU1otqs4pWUOyvdxLPKIk6P5WCJWp+AFJig=="
# ];

60
clusters/stokes/default.nix
View File

@@ -1,8 +1,8 @@
let
# Pin the deployment package-set to a specific version of nixpkgs
# pkgs = import (builtins.fetchTarball {
# url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
# sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
# url = "https://github.com/NixOS/nixpkgs/archive/e6377ff35544226392b49fa2cf05590f9f0c4b43.tar.gz";
# sha256 = "1fra9wwy5gvj5ibayqkzqpwdf715bggc0qbmrfch4fghwvl5m70l";
# }) {};
pkgs = import <nixpkgs> {};
@@ -24,6 +24,7 @@ let
map (n: ({ name = "c0-${toString n}"; address = "10.1.61.10${toString n}"; })) nodes;
stokes = {
# deployment.tags = [ "frontend" ];
node.myvnc = true;
systemd.targets = {
@@ -162,8 +163,6 @@ let
};
};
security.pam.services.sshd.googleAuthenticator.enable = true;
nix.extraOptions = ''
secret-key-files = /etc/nix/stokes.private
'';
@@ -209,10 +208,49 @@ let
};
};
# services.gitlab-runner = {
# enable = true;
# extraPackages = with pkgs; [
# singularity
# ];
# concurrent = 4;
# services = {
# sif = {
# registrationConfigFile = "/var/lib/secrets/gitlab-runner-registration";
# executor = "shell";
# tagList = [ "stokes" "sif" ];
# };
# };
# };
# security.sudo.extraConfig = ''
# gitlab-runner ALL=(ALL) NOPASSWD: /run/current-system/sw/bin/singularity
# '';
security.pam = {
services.sshd.googleAuthenticator.enable = true;
loginLimits = [
{
domain = "@users";
item = "rss";
type = "hard";
value = 16000000;
}
{
domain = "@users";
item = "cpu";
type = "hard";
value = 180;
}
];
};
imports = [ ./cluster.nix ./hw/frontend.nix ];
};
compute = {
# deployment.tags = [ "compute" ];
fileSystems = {
"/home/stokes" = {
device = "10.1.63.100:/home";
@@ -273,6 +311,16 @@ let
}
// compute;
};
in
{ inherit stokes; } // builtins.foldl' (a: n: a // mkCompute n) {} nodes
in {
## morph
# network = {
# inherit pkgs;
# description = "stokes";
# ordering = {
# tags = [ "frontend" "compute" ];
# };
# };
inherit stokes;
} // builtins.foldl' (a: n: a // mkCompute n) {} nodes

319
clusters/stokes/morph.nix
View File

@@ -1,319 +0,0 @@
let
# Pin the deployment package-set to a specific version of nixpkgs
# pkgs = import (builtins.fetchTarball {
# url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
# sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
# }) {};
pkgs = import <nixpkgs> {};
etcdNodes = {
# hpc0-0 = "10.1.63.100";
# hpc0-1 = "10.1.63.101";
# hpc0-2 = "10.1.63.102";
};
etcdCluster = {
enable = false;
existing = false;
nodes = etcdNodes;
};
k8sNodes = [
# { name = "hpc0-1"; address = "10.1.61.101"; }
];
stokes = {
deployment.tags = [ "frontend" ];
node.myvnc = true;
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
hibernate.enable = false;
hybrid-sleep.enable = false;
};
features = {
host = {
address = "10.1.61.100";
name = "hpc0-0";
};
os = {
externalInterface = "eno1";
nfs.enable = true;
nfs.exports = ''
/exports 10.1.61.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
/exports 10.1.63.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
'';
};
hpc = {
slurm.server = true;
frontend = true;
};
k8s = {
enable = true;
master.enable = true;
node.enable = true;
nodes = nodes;
clusterName = "hpc0";
initca = ./ca;
cidr = "10.100.0.0/16";
master = {
name = "hpc0-0";
address = "10.1.63.100";
extraSANs = [ "stokes.regnekraft.io" ];
};
ingressNodes = [
"hpc0-0.itpartner.intern"
];
fileserver = "mds0-0";
charts = {
acme_email = "innovasjon@itpartner.no";
grafana_smtp_user = "utvikling";
grafana_smtp_password = "S0m3rp0m@de#21!";
};
};
monitoring = {
server = {
enable = true;
scrapeHosts = [
"frontend" "mds0-0"
"c0-1" "c0-2" "c0-3" "c0-4" "c0-5" "c0-6" "c0-7" "c0-8"
];
defaultAlertReceiver = {
email_configs = [
{ to = "jonas.juselius@tromso.serit.no"; }
];
};
pageAlertReceiver = {
webhook_configs = [
{
url = "https://prometheus-msteams.k2.itpartner.no/stokes";
http_config = {
tls_config = { insecure_skip_verify = true; };
};
}
];
};
};
webUI.enable = true;
webUI.acmeEmail = "innovasjon@itpartner.no";
webUI.allow = [
"10.1.2.0/24"
"172.19.254.0/24"
"172.19.255.0/24"
];
infiniband-exporter = {
enable = true;
nameMap = ''
0x0c42a10300ddc4bc "frontend"
0x1c34da0300787798 "mds0-0"
0x0c42a10300dbe7f4 "c0-1"
0x0c42a10300dbe7d8 "c0-2"
0x0c42a10300dbe800 "c0-3"
0x0c42a10300dbec80 "c0-4"
0x0c42a10300dbea50 "c0-5"
0x0c42a10300dbeb2c "c0-6"
0x0c42a10300dbe7fc "c0-7"
0x0c42a10300dbe5a0 "c0-8"
'';
};
slurm-exporter = {
enable = true;
port = 6080;
};
};
};
networking = {
useDHCP = false;
interfaces.eno1 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.1.62.2";
prefixLength = 24;
} ];
};
interfaces.enp175s0f0 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.1.61.100";
prefixLength = 24;
} ];
};
interfaces.ibp59s0 = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.1.63.100";
prefixLength = 24;
} ];
};
defaultGateway = "10.1.62.1";
firewall.extraCommands = ''
iptables -I INPUT -s 10.1.63.0/24 -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.1.63.0/24 -j MASQUERADE
'';
};
fileSystems ={
"/exports/home" = {
device = "/home";
options = [ "bind" ];
};
"/exports/opt" = {
device = "/opt";
options = [ "bind" ];
};
"/data" = {
device = "10.1.63.80:/data";
fsType = "nfs";
};
};
security.pam.services.sshd.googleAuthenticator.enable = true;
nix.extraOptions = ''
secret-key-files = /etc/nix/stokes.private
'';
services.xserver = {
enable = true;
enableCtrlAltBackspace = true;
layout = "us";
xkbVariant = "altgr-intl";
xkbOptions = "eurosign:e";
displayManager = {
gdm.enable = true;
job.logToFile = true;
};
desktopManager.xfce.enable = true;
};
services.prometheus.alertmanager.configuration.global = {
smtp_smarthost = "smtpgw.itpartner.no:465";
smtp_auth_username = "utvikling";
smtp_auth_password = "S0m3rp0m@de#21!";
smtp_hello = "stokes.regnekraft.io";
smtp_from = "noreply@stokes.regnekraft.io";
};
services.nginx = {
virtualHosts = {
"ds.matnoc.regnekraft.io" = {
forceSSL = true;
enableACME = true;
serverAliases = [];
locations."/" = {
proxyPass = "http://localhost:9088";
proxyWebsockets = false;
extraConfig = ''
allow 10.1.2.0/24;
allow 172.19.254.0/24;
allow 172.19.255.0/24;
deny all;
'';
};
};
};
};
services.minio = {
enable = true;
region = "hpc";
browser = true;
accessKey = "admin";
secretKey = "en to tre fire";
listenAddress = "0.0.0.0:9000";
dataDir = [ "/data/minio" ];
};
imports = [ ./cluster.nix ./hw/frontend.nix ];
};
compute = {
deployment.tags = [ "compute" ];
features = {
os.externalInterface = "eno33";
hpc.compute = true;
};
fileSystems = {
"/home/stokes" = {
device = "10.1.63.100:/home";
fsType = "nfs";
};
"/opt" = {
device = "10.1.63.100:/opt";
fsType = "nfs";
};
"/data" = {
device = "10.1.63.80:/data";
fsType = "nfs";
};
};
};
genComputeNodes = idx: nNodes:
let
nodeList = builtins.genList (x: x + 1) nNodes;
mkCompute = n:
let
ip = "10.1.61.${toString (n + 100)}";
ipoib = "10.1.63.${toString (n + 100)}";
name = "c${toString idx}-${toString n}";
k8sName = "hpc${toString idx}-${toString n}";
hw = ./hw + "/${name}.nix";
in {
"${name}" = {
node = {
i40efix = true;
};
features.host = {
address = ip;
name = k8sName;
};
networking = {
useDHCP = false;
interfaces.eno33 = {
useDHCP = false;
ipv4.addresses = [ {
address = ip;
prefixLength = 24;
} ];
ipv4.routes = [ {
address = "10.1.62.2";
prefixLength = 32;
via = "10.1.61.100";
} ];
};
interfaces.ibp65s0 = {
useDHCP = false;
ipv4.addresses = [ {
address = ipoib;
prefixLength = 24;
} ];
};
};
imports = [ ./cluster.nix hw ];
} // compute;
};
in
builtins.foldl' (a: n: a // mkCompute n) {} nodeList;
in
{
network = {
inherit pkgs;
description = "stokes";
ordering = {
tags = [ "frontend" "compute" ];
};
};
inherit stokes;
} // genComputeNodes 0 8

21
clusters/stokes/users.nix
View File

@@ -18,6 +18,7 @@
yugaos = { gid = 1013; };
ata = { gid = 1014; };
kvile ={ gid = 1015; };
achim ={ gid = 1016; };
# @grp@
sif = {
@@ -37,6 +38,7 @@
"qin"
"yugaos"
"ata"
"achim"
];
};
@@ -94,6 +96,7 @@
createHome = false;
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAlfc2r3mNkvmdta+H/5zfdFe6317zmCdhhPYbipaGVFPUZO2cCTgSso28oDvOpCDldo/wl3jUxYNDlwH8LYMqKT3aGaOZr8JbxYzd+L+5GM2KTD+4YRmPtpYS/LWcc3j+fiFXSgX6Mrrgf6ineCRuBxSooDVE+pBakM1U7d5NE25apaAvclzFTmZBg0Sf9e5sgHkR99r9DUeGEQWGNZVUGwti39dFVp+aC9dsA+1/OtNB/HMF5G1MMk9dqvN7n7i9o9Plef2DParn4QU1GhmUKeEiBe4OAmSP+WwD4YvK6iXSKZG6tuTEspw+mR3rK5gBHrEiaNlCtp7O9BnAw4Wjhw== rsa-key-20201218"
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCp3QEG9uJq4EKMRWlBkqg/EQD6+2E90E6/1i+JLbiBYBSV6Yqac6KzyJezaVXzch+qm6FFXas3thaBScbA4cELkETaxyNzYG6sNcLyEhlQn51pEsq2mFiq8IiaMaDc55/2HTbXVrpNoTNQFt6lBwHziKQYuUI0H0cxze+ppp0ZJOu1MsayW8JOv75YSv6WFIDRR+KP6dOEBu1PsP6plTZwK94ogjQ+3KHGcMAnE3cf8VCEF8akNC6GRmCgXNZE4I9MmKTDr217OoFpnXAx5/KTvGo+USkXc6xn/vbQsni4ExwGlMTg97RK49wIHD1NfGxZ3sv7mZ+UQPqqmSxCG+zueJrR6BSBfbm7fw5KvRn69rOihapeo/6GoqqVDe4yn1imtojjHN6+9pgJ9E6o108qbXRw2X6t1KUuXrB+fTfUKvy0kWiJFIMDSUtKF/nhiES+aCI4b4WBwyg5hdKGvgJdjyUS7P/jYgqWRe+qmknAERtQKlFDA/C6ChsTXerFD5Ikvu3dajJUiDehszEON5F4JlxSf2VpUFCDLVNqV/GjJqOg90mXGDk82c+0ZHIUsPLsdqR+t/xnOSv1Ks9I5fId6g+3OlR1ifnb3Qm48QGKbi/CM8M/QXzv4VgeIkRTR0Oi4W0P1tpUSyPd1nyGaM/B/FqN52XIUjRqIfu0emwgiw== olean@navier"
];
};
@@ -319,6 +322,24 @@
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtB+HWtE4iXJiRVi1MUKaE3R3FAcHzCgiF84ho6GXKxx5H2iY8sgfxWo/lFSonhZKTo/+dHOYNKs42Q85ytG1rpcEYYVOK53mx8f7Z3THmw348a/+geM8Bukvo5pLc7KmXIvq6UQIjZmI/wnbA7B8MzLyrod71SaT1ujMEV1Jg0b3KnjS5kJnUHDICw3CdvuenNIgYl/zbTeEJ1iUu6T1TY+cNGG/7HOsaR1leCArDutHIKowcIFQFZoLEikM2DX5MSp9UBizAVogHugEqE2Bqh+C7NyTzJfQzR8s4drnt9IaptJQmCo6z9f+dQALjhftJXBDdkR6coMyOujV3Yyc5 rsa-key-20210928"
];
};
achim = {
description = "Achim Randelhoff";
home = "/home/achim";
group = "achim";
extraGroups = [
"users"
];
uid = 1016;
isNormalUser = true;
createHome = false;
useDefaultShell = true;
# shell = pkgs.fish;
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuShthrciA4hz3g/e4Q2S1L8OOyb0BpXg3FdjAhBHq8ZWK3GD40qyjmvk45EMX+2hZBXKKjEgO2ToTnH6P0NggwfBU9XhEb/SVxJwohJs55xRERT8jTXcYAXGrZntUg79ndWUHL2NzMMSJnJPEX1M3GZIymDxmUzsaagNvRI3kja42FNHtdX49hGSSygRoqjE+ui2lbFVi6+uY8TUdeW03+BYOgOJ8AtbvwP8MDZqUbHWc7fbg1DE3n52i+Uje2xyXPRwgCKZ0Ha0OLwiezKkVlUqc2gzSIQlKZ2Oy+9AE1knbCr5LVsarERUc17ux74fNQF8P6mCbbqvsgpX0KJK4yrjXPvkFVLcqmRXG+wyYuLLIAuNSG9N6rDgelBevTIqH+zZusXJm2du7mATgpEKBjYHlyS4tuf+gJaP26A2E1Eay5xxUKawm/PY71g/nMHlifYlnbz7fexQ/ObKCntLC0PP07xA6X8einCO81Q+8y0upa4hyzMkfHN4hcknXvj0= doppler@AKVA9454
"
];
};
# @usr@
};