oceanbox/platform
6
0
Fork 0
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

unified cluster setup using colmena

Browse Source
This commit is contained in:
Jonas Juselius
2022-09-09 11:00:55 +02:00
parent ff19c52fd1
commit 980364adee
61 changed files with 966 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
configuration.nix
View File

@@ -1,11 +1,10 @@
{ pkgs, ...}:
let
nodes = import ./nixops/ekman/nodes.nix;
myvnc = import ./nixops/ekman/myvnc.nix { inherit pkgs; };
computeNodes = import ./cluster/compute/nodes.nix;
etcdCluster = import ./cluster/etcdCluster.nix;
myvnc = import ./cluster/compute/myvnc.nix { inherit pkgs; };
in
{
# deployment.tags = [ "frontend" ];
systemd.targets = {
sleep.enable = false;
suspend.enable = false;
@@ -40,13 +39,15 @@ in
k8s = {
master.enable = true;
inherit nodes;
node.enable = true;
nodes = computeNodes;
inherit etcdCluster;
};
monitoring = {
server = {
enable = false;
scrapeHosts = [ "frontend" "mds1-0" ] ++ (builtins.map (x: x.name) nodes);
scrapeHosts = [ "frontend" "nfs0" "nfs1" ] ++ (builtins.map (x: x.name) computeNodes);
defaultAlertReceiver = {
email_configs = [
{ to = "jonas.juselius@oceanbox.io"; }
@@ -76,6 +77,22 @@ in
0xe8ebd3030024a2c6 "frontend"
0xe8ebd3030024981e "c0-1"
0xe8ebd3030024a21a "c0-2"
0xe8ebd30300249a3a "c0-3"
0xe8ebd30300248b9e "c0-4"
0xe8ebd30300248b86 "c0-5"
0xe8ebd3030024998a "c0-6"
0xe8ebd30300248b8e "c0-7"
0xe8ebd3030024999e "c0-8"
0xe8ebd30300248fca "c0-9"
0xe8ebd3030024a216 "c0-10"
0xe8ebd30300248b96 "c0-11"
0xe8ebd30300248b9a "c0-12"
0xe8ebd303002495d2 "c0-13"
0xe8ebd303002495e2 "c0-14"
0xe8ebd30300248f42 "c0-15"
0xe8ebd303002495e6 "c0-16"
0xe8ebd3030024a2a2 "nfs0"
0xe8ebd3030024a2ae "nfs1"
'';
};
slurm-exporter = {
@@ -85,6 +102,10 @@ in
};
};
services.udev.extraRules = ''
KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
'';
networking = {
useDHCP = false;
hostName = "ekman";
@@ -101,8 +122,16 @@ in
address = "10.255.241.100";
prefixLength = 24;
} ];
# ipv4.routes = [ {
# address = "10.1.2.0";
# prefixLength = 24;
# via = "10.255.241.1";
# } ];
};
interfaces.ibp65s0 = {
interfaces."ibp65s0" = {
useDHCP = false;
};
interfaces."ibp65s0.7666" = {
useDHCP = false;
ipv4.addresses = [ {
address = "10.255.243.100";
@@ -110,10 +139,15 @@ in
} ];
};
defaultGateway = "10.255.242.1";
firewall.extraCommands = ''
iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';
firewall = {
allowedTCPPorts = [ 4443 ];
extraCommands = ''
# iptables -I INPUT -s 10.1.2.0/22 -j ACCEPT
iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
'';
};
};
fileSystems = {
@@ -217,6 +251,18 @@ in
];
};
system.activationScripts = {
home-permissions.text = ''
chmod 755 /home/olean
chmod 755 /home/frankgaa
chmod 755 /home/jonas
chmod 755 /home/stig
chmod 755 /home/bast
chmod 755 /home/simenlk
'';
};
# ssh-rsa is deprecated, but putty/winscp users use it
# services.openssh.extraConfig = ''
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
@@ -224,6 +270,6 @@ in
environment.systemPackages = [ myvnc ];
imports = [ ./nixops/ekman/cluster.nix ./hardware-configuration.nix ];
imports = [ ./cluster/cluster.nix ./hardware-configuration.nix ];
}