major: grand unified clusterfck (ekman not tested yet)
This commit is contained in:
Jonas Juselius
354
ekman/manage/default.nix
Normal file
354
ekman/manage/default.nix
Normal file
@@ -0,0 +1,354 @@
|
||||
<<<<<<< HEAD:configuration.nix
|
||||
_:
|
||||
{
|
||||
imports = [ ./rossby/manage ];
|
||||
=======
|
||||
{ pkgs, ...}:
|
||||
let
|
||||
computeNodes =
|
||||
import ../c0/nodes.nix ++
|
||||
import ../c1/nodes.nix ++
|
||||
[
|
||||
rec {
|
||||
idx = 100;
|
||||
name = "ekman";
|
||||
address = "10.255.241.${toString idx}";
|
||||
ipoib = "10.255.243.${toString idx}";
|
||||
pubkey = ../login/ekman.pub;
|
||||
}
|
||||
rec {
|
||||
idx = 90;
|
||||
name = "fs-work";
|
||||
address = "10.255.241.${toString idx}";
|
||||
ipoib = "10.255.243.${toString idx}";
|
||||
pubkey = ../fs-work/fs-work.pub;
|
||||
}
|
||||
rec {
|
||||
idx = 81;
|
||||
name = "fs-backup";
|
||||
address = "10.255.241.${toString idx}";
|
||||
ipoib = "10.255.243.${toString idx}";
|
||||
pubkey = ../fs-backup/fs-backup.pub;
|
||||
}
|
||||
];
|
||||
etcdCluster = import ../etcdCluster.nix;
|
||||
name = "ekman-manage";
|
||||
address = "10.255.241.99";
|
||||
ipoib = "10.255.243.99";
|
||||
in {
|
||||
systemd.targets = {
|
||||
sleep.enable = false;
|
||||
suspend.enable = false;
|
||||
hibernate.enable = false;
|
||||
hybrid-sleep.enable = false;
|
||||
};
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
rdma-core
|
||||
hwloc
|
||||
headscale
|
||||
];
|
||||
|
||||
cluster = {
|
||||
k8sNode = true;
|
||||
compute = false;
|
||||
slurm = true;
|
||||
mounts = {
|
||||
rdma.enable = true;
|
||||
automount.enable = true;
|
||||
home = true;
|
||||
opt = true;
|
||||
work = true;
|
||||
data = true;
|
||||
backup = true;
|
||||
ceph = true;
|
||||
};
|
||||
};
|
||||
|
||||
features = {
|
||||
desktop.enable = false;
|
||||
cachix.enable = false;
|
||||
|
||||
host = {
|
||||
inherit address;
|
||||
inherit name;
|
||||
};
|
||||
|
||||
myvnc.enable = false;
|
||||
|
||||
os = {
|
||||
externalInterface = "eno1";
|
||||
nfs.enable = false;
|
||||
nfs.exports = ''
|
||||
/exports 10.255.241.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
/exports 10.255.243.0/24(insecure,rw,async,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
hpc = {
|
||||
slurm.server = true;
|
||||
slurm.slurmrestd = false;
|
||||
slurm.mungeUid = 996;
|
||||
frontend = true;
|
||||
};
|
||||
|
||||
k8s = {
|
||||
master.enable = true;
|
||||
node.enable = true;
|
||||
nodes = computeNodes;
|
||||
inherit etcdCluster;
|
||||
};
|
||||
|
||||
monitoring = {
|
||||
server = {
|
||||
enable = false;
|
||||
scrapeHosts = [
|
||||
"ekman-manage"
|
||||
"ekman"
|
||||
"fs-work"
|
||||
"fs-backup"
|
||||
] ++ (builtins.map (x: x.name) computeNodes);
|
||||
defaultAlertReceiver = {
|
||||
email_configs = [
|
||||
{ to = "jonas.juselius@oceanbox.io"; }
|
||||
];
|
||||
};
|
||||
pageAlertReceiver = {
|
||||
webhook_configs = [
|
||||
{
|
||||
url = "https://prometheus-msteams.k2.itpartner.no/ekman";
|
||||
http_config = {
|
||||
tls_config = { insecure_skip_verify = true; };
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
webUI.enable = false;
|
||||
webUI.acmeEmail = "innovasjon@itpartner.no";
|
||||
webUI.allow = [
|
||||
"10.1.2.0/24"
|
||||
"172.19.254.0/24"
|
||||
"172.19.255.0/24"
|
||||
];
|
||||
infiniband-exporter = {
|
||||
enable = true;
|
||||
nameMap = ''
|
||||
0xe8ebd3030024a2c6 "ekman"
|
||||
0x0c42a10300ddc4bc "ekman-manage"
|
||||
0xe8ebd3030024a2ae "fs-work"
|
||||
0x1c34da0300787798 "fs-backup"
|
||||
0xe8ebd3030024981e "c0-1"
|
||||
0xe8ebd3030024a21a "c0-2"
|
||||
0xe8ebd30300249a3a "c0-3"
|
||||
0xe8ebd30300248b9e "c0-4"
|
||||
0xe8ebd30300248b86 "c0-5"
|
||||
0xe8ebd3030024998a "c0-6"
|
||||
0xe8ebd30300248b8e "c0-7"
|
||||
0xe8ebd3030024999e "c0-8"
|
||||
0xe8ebd30300248fca "c0-9"
|
||||
0xe8ebd3030024a216 "c0-10"
|
||||
0xe8ebd30300248b96 "c0-11"
|
||||
0xe8ebd30300248b9a "c0-12"
|
||||
0xe8ebd303002495d2 "c0-13"
|
||||
0xe8ebd303002495e2 "c0-14"
|
||||
0xe8ebd30300248f42 "c0-15"
|
||||
0xe8ebd303002495e6 "c0-16"
|
||||
0x0c42a10300dbe7f4 "c1-1"
|
||||
0x0c42a10300dbe7d8 "c1-2"
|
||||
0x0c42a10300dbe800 "c1-3"
|
||||
0x0c42a10300dbec80 "c1-4"
|
||||
0x0c42a10300dbea50 "c1-5"
|
||||
0x0c42a10300dbeb2c "c1-6"
|
||||
0x0c42a10300dbe7fc "c1-7"
|
||||
0x0c42a10300dbe5a0 "c1-8"
|
||||
'';
|
||||
};
|
||||
slurm-exporter = {
|
||||
enable = true;
|
||||
port = 6080;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
programs.singularity.enable = true;
|
||||
|
||||
# services.udev.extraRules = ''
|
||||
# KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
# '';
|
||||
|
||||
services.kubernetes.apiserver.extraOpts = ''--oidc-client-id=9b6daef0-02fa-4574-8949-f7c1b5fccd15 --oidc-groups-claim=roles --oidc-issuer-url=https://login.microsoftonline.com/3f737008-e9a0-4485-9d27-40329d288089/v2.0'';
|
||||
|
||||
services.flannel.iface = "eno2";
|
||||
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
hostName = name;
|
||||
interfaces.eno1 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = "10.255.242.3";
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
interfaces.eno2 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [
|
||||
{
|
||||
inherit address;
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
interfaces.ens2f1np1 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [
|
||||
{
|
||||
address = "10.255.244.99";
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
interfaces.ibs2f0 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [
|
||||
{
|
||||
address = ipoib;
|
||||
prefixLength = 24;
|
||||
}
|
||||
];
|
||||
};
|
||||
defaultGateway = "10.255.242.1";
|
||||
firewall = {
|
||||
allowedTCPPorts = [ 4443 4725 ];
|
||||
extraCommands = ''
|
||||
# needed for nodeport access on k1 and k2
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.241.0/24 ! -d 10.255.0.0/16 -j SNAT --to-source 10.255.242.3
|
||||
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -s 100.64.0.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -d 10.255.244.0/24 -j MASQUERADE
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -d 10.255.241.0/16 -j SNAT --to-source 10.255.241.99
|
||||
# iptables -t nat -A POSTROUTING -s 10.255.244.0/24 -j SNAT --to-source 10.255.242.3
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/exports/public" = {
|
||||
device = "/srv/public";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
};
|
||||
|
||||
nix.extraOptions = ''
|
||||
# secret-key-files = /etc/nix/ekman.key
|
||||
'';
|
||||
|
||||
services.prometheus.alertmanager.configuration.global = {
|
||||
smtp_smarthost = "smtpgw.itpartner.no";
|
||||
# smtp_auth_username = "utvikling";
|
||||
# smtp_auth_password = "S0m3rp0m@de#21!";
|
||||
smtp_hello = "ekman.oceanbox.io";
|
||||
smtp_from = "noreply@ekman.oceanbox.io";
|
||||
};
|
||||
|
||||
security.pam = {
|
||||
services.sshd.googleAuthenticator.enable = true;
|
||||
loginLimits = [
|
||||
{
|
||||
domain = "@users";
|
||||
item = "rss";
|
||||
type = "hard";
|
||||
value = 16000000;
|
||||
}
|
||||
{
|
||||
domain = "@users";
|
||||
item = "cpu";
|
||||
type = "hard";
|
||||
value = 180;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
system.activationScripts = {
|
||||
home-permissions.text = ''
|
||||
chmod 755 /home/olean
|
||||
chmod 755 /home/frankgaa
|
||||
chmod 755 /home/jonas
|
||||
chmod 755 /home/stig
|
||||
chmod 755 /home/bast
|
||||
chmod 755 /home/mrtz
|
||||
chmod 755 /home/avle
|
||||
chmod 755 /home/simenlk
|
||||
chmod 755 /home/ole
|
||||
'';
|
||||
};
|
||||
|
||||
# Use nvd to get package diff before apply
|
||||
system.activationScripts.system-diff = {
|
||||
supportsDryActivation = true; # safe: only outputs to stdout
|
||||
text = ''
|
||||
export PATH="${pkgs.lib.makeBinPath [ pkgs.nixVersions.latest ]}:$PATH"
|
||||
if [ -e /run/current-system ]; then
|
||||
${pkgs.lib.getExe pkgs.nvd} diff '/run/current-system' "$systemConfig" || true
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
# ssh-rsa is deprecated, but putty/winscp users use it
|
||||
services.openssh.extraConfig = ''
|
||||
# pubkeyacceptedalgorithms ssh-rsa,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
|
||||
PubkeyAuthOptions verify-required
|
||||
'';
|
||||
|
||||
# boot.kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
|
||||
|
||||
virtualisation.docker.enable = pkgs.lib.mkForce true;
|
||||
|
||||
# Configuration for the coordination server for a tailscale network run using headscale.
|
||||
#
|
||||
# We can set it up to provide several exit nodes through which traffic can be routed.
|
||||
#
|
||||
# Servers can join using this command:
|
||||
# `tailscale up --login-server net.b0.itpartner.no --accept-dns=false --advertise-exit-node`
|
||||
#
|
||||
# with the following config:
|
||||
#
|
||||
# service.tailscale = {
|
||||
# enable = true;
|
||||
# useRoutingFeatures = "server"; # for exit-node usage
|
||||
# };
|
||||
#
|
||||
# Clients can join using this command:
|
||||
# `tailscale up --login-server net.b0.itpartner.no --accept-dns=false`
|
||||
#
|
||||
# services.headscale = {
|
||||
# enable = true;
|
||||
# address = "0.0.0.0";
|
||||
# port = 4725; # hscl
|
||||
# settings = import ./headscale/settings.nix;
|
||||
# };
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
authKeyFile = "/var/lib/secrets/tailscale.key";
|
||||
useRoutingFeatures = "both"; # for exit-node usage
|
||||
extraUpFlags = [
|
||||
"--login-server=https://headscale.svc.oceanbox.io"
|
||||
"--accept-dns=false"
|
||||
"--advertise-exit-node"
|
||||
];
|
||||
};
|
||||
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
../default.nix
|
||||
../mounts.nix
|
||||
../myvnc.nix
|
||||
];
|
||||
>>>>>>> ekman:ekman/manage/default.nix
|
||||
}
|
||||
27
ekman/manage/hardware-configuration.nix
Normal file
27
ekman/manage/hardware-configuration.nix
Normal file
@@ -0,0 +1,27 @@
|
||||
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||
# and may be overwritten by future invocations. Please make changes
|
||||
# to /etc/nixos/configuration.nix instead.
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports =
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-intel" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/099be829-726d-44c9-b113-4c2604a663c5";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/8111-9F24";
|
||||
fsType = "vfat";
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
}
|
||||
1
ekman/manage/ssh_host_key.pub
Normal file
1
ekman/manage/ssh_host_key.pub
Normal file
@@ -0,0 +1 @@
|
||||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkSSuHkieXwgFMKRy4MjwjNrJEItWbQHeAAH+Zn1YZp root@localhost
|
||||
Reference in New Issue
Block a user