unified cluster setup using colmena
This commit is contained in:
Jonas Juselius
202
cluster/nfs0/default.nix
Normal file
202
cluster/nfs0/default.nix
Normal file
@@ -0,0 +1,202 @@
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
let
|
||||
# Pin the deployment package-set to a specific version of nixpkgs
|
||||
# pkgs = import (builtins.fetchTarball {
|
||||
# url = "https://github.com/NixOS/nixpkgs/archive/e9148dc1c30e02aae80cc52f68ceb37b772066f3.tar.gz";
|
||||
# sha256 = "1ckzhh24mgz6jd1xhfgx0i9mijk6xjqxwsshnvq789xsavrmsc36";
|
||||
# }) {};
|
||||
# pkgs = import <nixpkgs> {};
|
||||
etcdCluster = import ../etcdCluster.nix;
|
||||
name = "nfs0";
|
||||
address = "10.255.241.80";
|
||||
in {
|
||||
nfs0 = { config, pkgs, ... }: with pkgs; {
|
||||
deployment.tags = [ "fs" "fs-data" ];
|
||||
deployment.targetHost = address;
|
||||
system.autoUpgrade.enable = lib.mkForce false;
|
||||
|
||||
systemd.targets = {
|
||||
sleep.enable = false;
|
||||
suspend.enable = false;
|
||||
hibernate.enable = false;
|
||||
hybrid-sleep.enable = false;
|
||||
};
|
||||
|
||||
environment.etc = {
|
||||
"minio/rootcredentials" = {
|
||||
text = ''
|
||||
accessKey="admin"
|
||||
secretKey="en to tre fire"
|
||||
'';
|
||||
mode = "600";
|
||||
uid = 280;
|
||||
};
|
||||
};
|
||||
|
||||
services.udev.extraRules = ''
|
||||
KERNEL=="ibp65s0", SUBSYSTEM=="net", ATTR{create_child}:="0x7666"
|
||||
'';
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
rdma-core
|
||||
hwloc
|
||||
];
|
||||
|
||||
cluster.k8sNode = true;
|
||||
|
||||
features = {
|
||||
host = {
|
||||
inherit address;
|
||||
inherit name;
|
||||
};
|
||||
|
||||
os = {
|
||||
externalInterface = "enp33s0f3np3";
|
||||
nfs.enable = true;
|
||||
nfs.exports = ''
|
||||
/exports 10.255.241.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
/exports 10.255.243.0/24(insecure,rw,sync,no_subtree_check,crossmnt,fsid=0,no_root_squash)
|
||||
'';
|
||||
};
|
||||
|
||||
k8s = {
|
||||
enable = true;
|
||||
node.enable = true;
|
||||
master.enable = false;
|
||||
inherit etcdCluster;
|
||||
};
|
||||
};
|
||||
|
||||
# certs = {
|
||||
# enable = true;
|
||||
# caBundle = ./ca;
|
||||
# certs = [
|
||||
# {
|
||||
# name = name;
|
||||
# SANs = [ "${name}.cluster.local" address ];
|
||||
# owner = "nginx";
|
||||
# group = "nginx";
|
||||
# }
|
||||
# ];
|
||||
# };
|
||||
|
||||
systemd.services.rc-local = {
|
||||
description = "rc.local script";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
path = [ "/run/current-system/sw/" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
};
|
||||
script = ''
|
||||
if [ -e /sys/block/md126 ]; then
|
||||
echo "deadline" > /sys/block/md126/queue/scheduler
|
||||
# echo "4096" > /sys/block/md126/queue/nr_requests
|
||||
echo "4096" > /sys/block/md126/queue/read_ahead_kb
|
||||
echo "always" > /sys/kernel/mm/transparent_hugepage/enabled
|
||||
echo "always" > /sys/kernel/mm/transparent_hugepage/defrag
|
||||
fi
|
||||
grep -q rdma /proc/fs/nfsd/portlist || echo "rdma 20049" > /proc/fs/nfsd/portlist
|
||||
'';
|
||||
};
|
||||
|
||||
boot.kernel.sysctl = {
|
||||
"vm.dirty_background_ratio" = 5;
|
||||
"vm.dirty_ratio" = 10;
|
||||
"vm.vfs_cache_pressure" = 50;
|
||||
"vm.min_free_kbytes" = 262144;
|
||||
};
|
||||
|
||||
services.minio = {
|
||||
enable = true;
|
||||
region = "store1";
|
||||
browser = true;
|
||||
rootCredentialsFile = "/etc/minio/rootcredentials";
|
||||
listenAddress = "0.0.0.0:9000";
|
||||
dataDir = [ "/data/s3" ];
|
||||
};
|
||||
|
||||
networking = {
|
||||
hostName = name;
|
||||
interfaces.enp33s0f3np3 = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = address;
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
ipv4.routes = [ {
|
||||
address = "10.255.242.2";
|
||||
prefixLength = 32;
|
||||
via = "10.255.241.100";
|
||||
} ];
|
||||
};
|
||||
interfaces."ibp65s0" = {
|
||||
useDHCP = false;
|
||||
};
|
||||
interfaces."ibp65s0.7666" = {
|
||||
useDHCP = false;
|
||||
ipv4.addresses = [ {
|
||||
address = "10.255.243.80";
|
||||
prefixLength = 24;
|
||||
} ];
|
||||
};
|
||||
firewall = {
|
||||
allowedTCPPorts = [ 443 9000 9001 ];
|
||||
allowedUDPPorts = [];
|
||||
extraCommands = ''
|
||||
iptables -I INPUT -s 10.255.241.0/24 -j ACCEPT
|
||||
iptables -I INPUT -s 10.255.243.0/24 -j ACCEPT
|
||||
iptables -t nat -A POSTROUTING -s 10.255.243.0/24 -j MASQUERADE
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# services.nginx = {
|
||||
# enable = true;
|
||||
# statusPage = false;
|
||||
# virtualHosts = {
|
||||
# "s3ui.oceanbox.io" = {
|
||||
# forceSSL = true;
|
||||
# enableACME = false;
|
||||
# sslTrustedCertificate = "/var/lib/secrets/ca.pem";
|
||||
# sslCertificate = "/var/lib/secrets/s3.pem";
|
||||
# sslCertificateKey = "/var/lib/secrets/s3-key.pem";
|
||||
# serverAliases = [];
|
||||
# locations."/" = {
|
||||
# proxyPass = "http://127.0.0.1:9001";
|
||||
# extraConfig = ''
|
||||
# allow all;
|
||||
# '';
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
|
||||
fileSystems = {
|
||||
"/exports/data" = {
|
||||
device = "/data";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/exports/ekman" = {
|
||||
device = "/data/nfs-provisioner";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/exports/opt" = {
|
||||
device = "/opt";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
"/vol/local-storage/vol1" = {
|
||||
device = "/vol/vol1";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
};
|
||||
|
||||
programs.singularity.enable = true;
|
||||
imports = [
|
||||
../cluster.nix
|
||||
./hardware-configuration.nix
|
||||
];
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
55
cluster/nfs0/hardware-configuration.nix
Normal file
55
cluster/nfs0/hardware-configuration.nix
Normal file
@@ -0,0 +1,55 @@
|
||||
# Do not modify this file! It was generated by ‘nixos-generate-config’
|
||||
# and may be overwritten by future invocations. Please make changes
|
||||
# to /etc/nixos/configuration.nix instead.
|
||||
{ config, lib, pkgs, modulesPath, ... }:
|
||||
|
||||
{
|
||||
imports =
|
||||
[ (modulesPath + "/installer/scan/not-detected.nix")
|
||||
];
|
||||
|
||||
boot.initrd.availableKernelModules = [ "megaraid_sas" "xhci_pci" "ahci" "usbhid" "sd_mod" ];
|
||||
boot.initrd.kernelModules = [ "dm-snapshot" ];
|
||||
boot.kernelModules = [ "kvm-amd" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" =
|
||||
{ device = "/dev/disk/by-uuid/46dfa481-ccd4-4e4f-b1d7-6875d582f7cd";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" =
|
||||
{ device = "/dev/disk/by-uuid/F71C-0BD2";
|
||||
fsType = "vfat";
|
||||
};
|
||||
|
||||
fileSystems."/data" =
|
||||
{ device = "/dev/disk/by-uuid/126c5d04-5266-43e8-887d-740d5944cb2b";
|
||||
fsType = "xfs";
|
||||
options = [
|
||||
"noatime"
|
||||
"nodiratime"
|
||||
"logbufs=8"
|
||||
"logbsize=256k"
|
||||
"largeio"
|
||||
"inode64"
|
||||
"swalloc"
|
||||
"allocsize=131072k"
|
||||
];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
|
||||
# (the default) this is the recommended approach. When using systemd-networkd it's
|
||||
# still possible to use this option, but it's recommended to use it in conjunction
|
||||
# with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.docker0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f0np0.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f1np1.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f2np2.useDHCP = lib.mkDefault true;
|
||||
# networking.interfaces.enp33s0f3np3.useDHCP = lib.mkDefault true;
|
||||
|
||||
hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
|
||||
}
|
||||
1
cluster/nfs0/nfs0.pub
Normal file
1
cluster/nfs0/nfs0.pub
Normal file
@@ -0,0 +1 @@
|
||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7wnYBwAdcp5+y7QOKzjd2apRv+Nvquy+qr3jVBeiQgbGsjlpGJBRsGv46QhSbB9QLuFuCUluuvilYbS9/ubm+BtRCrFViAHwFx+QhbLYlAXJPLGfXLxJZDSxbM6c9L+N+GAAfhiCfdhUVB39G33v5dd/VD3J4F/xC7lkPJU59pGokqe9dGrDta33pemSzR9QV9RuV4ZxJG1ld1B2L6TjSkAPTb79PPvMw8LnTViNffmHKp5qvwgIfwRvMApG5KnHw720pPncTW4ejY+utSbP4DcwD2qayKbxnsYDpimMDQxR3id6meZa5LcilxExmz/ig7Fpc3TB3wIpT4ha6VVu2FhRThKR5zD1Qif8BaKy5eAmqcWKQm6M9W56LmUdn/OHpzmaSv47UgtJ+1JG4LqGSaShs8kfVWtaqmev6vvnHBZMm7ni3xyROTSWnjJHLDQJbCYb1PBbct2Dod6MuU54CJUuxw0k+hCJ1QMz1ibme3+LauXVuLJ2JrnAoz7x2CcUv9e5MVx0H4HpWTCxi7cNBO5bbeuxMwHbl3JrZ58LBK9zZYl+nr2F7CZqLyoMimQuVpXT6Aw0k/nfYMAXjUJ3qbBldiGsRgpgbvL27nJK0EhQ8Kd36QhHfdJKhiAIPV+06WVWhPllaqay4OgZIQM1aB83eHg/OUxbOWEu89d7w/w== root@nixos
|
||||
Reference in New Issue
Block a user